gcp

sc_crawler.vendors.gcp

inventory_compliance_frameworks

inventory_compliance_frameworks(vendor)

Manual list of compliance frameworks known for GCP.

Resources: https://cloud.google.com/compliance?hl=en

Source code in sc_crawler/vendors/gcp.py

def inventory_compliance_frameworks(vendor):
    """Manual list of compliance frameworks known for GCP.

    Resources: <https://cloud.google.com/compliance?hl=en>"""
    return map_compliance_frameworks_to_vendor(
        vendor.vendor_id, ["hipaa", "soc2t2", "iso27001"]
    )

inventory_regions

inventory_regions(vendor)

List all available GCP regions via API calls.

Some data sources are not available from APIs, and were collected manually:

• location: https://cloud.google.com/compute/docs/regions-zones#available and https://en.wikipedia.org/wiki/Google_data_centers,
• lon/lat coordinates: https://en.wikipedia.org/wiki/Google_data_centers#Locations and approximation based on the city when no more accurate data was available.
• energy carbon data: https://cloud.google.com/sustainability/region-carbon#data and https://github.com/GoogleCloudPlatform/region-carbon-info,
• launch dates were collected from Wikipedia and GCP blog posts, such as https://medium.com/@retomeier/an-annotated-history-of-googles-cloud-platform-90b90f948920 and https://cloud.google.com/blog/products/infrastructure/introducing-new-google-cloud-regions.

Note that many GCP regions use more than 90% green energy, but the related flag in our database is set to False as not being 100%.

Source code in sc_crawler/vendors/gcp.py

def inventory_regions(vendor):
    """List all available GCP regions via API calls.

    Some data sources are not available from APIs, and were collected manually:

    - location: <https://cloud.google.com/compute/docs/regions-zones#available> and <https://en.wikipedia.org/wiki/Google_data_centers>,
    - lon/lat coordinates: <https://en.wikipedia.org/wiki/Google_data_centers#Locations> and approximation based on the city when no more accurate data was available.
    - energy carbon data: <https://cloud.google.com/sustainability/region-carbon#data> and <https://github.com/GoogleCloudPlatform/region-carbon-info>,
    - launch dates were collected from [Wikipedia](https://en.wikipedia.org/wiki/Google_Cloud_Platform#Regions_and_zones) and GCP blog posts, such as <https://medium.com/@retomeier/an-annotated-history-of-googles-cloud-platform-90b90f948920> and <https://cloud.google.com/blog/products/infrastructure/introducing-new-google-cloud-regions>.

    Note that many GCP regions use more than 90% green energy,
    but the related flag in our database is set to `False` as not being 100%.
    """

    manual_data = {
        "africa-south1": {
            "country_id": "ZA",
            "city": "Johannesburg",
            # https://cloud.google.com/blog/products/infrastructure/heita-south-africa-new-cloud-region
            "founding_year": 2024,
            "green_energy": False,
            # approximation based on city
            "lat": -26.0420631,
            "lon": 28.0589808,
        },
        "asia-east1": {
            "country_id": "TW",
            "state": "Changhua County",
            "founding_year": 2013,
            "green_energy": False,
            "lat": 24.1385,
            "lon": 120.425722,
        },
        "asia-east2": {
            "country_id": "HK",
            # https://cloud.google.com/blog/products/gcp/gcps-region-in-hong-kong-is-now-open
            "founding_year": 2018,
            "green_energy": False,
            # approximation based on country
            "lat": 22.2772377,
            "lon": 114.1703066,
            "display_name": "Hong Kong",
        },
        "asia-northeast1": {
            "country_id": "JP",
            "city": "Tokyo",
            "state": "Japan",
            "founding_year": 2016,
            "green_energy": False,
            # approximation based on city
            "lat": 35.6433846,
            "lon": 139.7684933,
        },
        "asia-northeast2": {
            "country_id": "JP",
            "city": "Osaka",
            "founding_year": 2019,
            "green_energy": False,
            # approximation based on city
            "lat": 34.6696646,
            "lon": 135.4846612,
        },
        "asia-northeast3": {
            "country_id": "KR",
            "city": "Seoul",
            "founding_year": 2020,
            "green_energy": False,
            # approximation based on city
            "lat": 37.5514982,
            "lon": 126.97784,
        },
        "asia-south1": {
            "country_id": "IN",
            "city": "Mumbai",
            "founding_year": 2017,
            "green_energy": False,
            # approximation based on city
            "lat": 19.0709441,
            "lon": 72.8726468,
        },
        "asia-south2": {
            "country_id": "IN",
            "city": "Delhi",
            "founding_year": 2021,
            "green_energy": False,
            # approximation based on city
            "lat": 28.6439839,
            "lon": 76.9284239,
        },
        "asia-southeast1": {
            "country_id": "SG",
            "city": "Jurong West",
            "founding_year": 2017,
            "green_energy": False,
            "lat": 1.351333,
            "lon": 103.709778,
        },
        "asia-southeast2": {
            "country_id": "ID",
            "city": "Jakarta",
            "founding_year": 2020,
            "green_energy": False,
            # approximation based on city
            "lat": -6.2297401,
            "lon": 106.747117,
        },
        "australia-southeast1": {
            "country_id": "AU",
            "city": "Sydney",
            "founding_year": 2017,
            "green_energy": False,
            # approximation based on city
            "lat": -33.8375583,
            "lon": 150.9488095,
        },
        "australia-southeast2": {
            "country_id": "AU",
            "city": "Melbourne",
            "founding_year": 2021,
            "green_energy": False,
            # approximation based on city
            "lat": -37.8038607,
            "lon": 144.7119569,
        },
        "europe-central2": {
            "country_id": "PL",
            "city": "Warsaw",
            "founding_year": 2021,
            "green_energy": False,
            # approximation based on city
            "lat": 52.2328871,
            "lon": 20.8966164,
        },
        "europe-north1": {
            "country_id": "FI",
            "city": "Hamina",
            "founding_year": 2018,
            "green_energy": False,
            "lat": 60.536578,
            "lon": 27.117003,
        },
        "europe-southwest1": {
            "country_id": "ES",
            "city": "Madrid",
            "founding_year": 2022,
            "green_energy": False,
            "lat": 40.519533,
            "lon": -3.340937,
        },
        "europe-west1": {
            "country_id": "BE",
            "city": "St. Ghislain",
            # https://medium.com/@retomeier/an-annotated-history-of-googles-cloud-platform-90b90f948920
            "founding_year": 2015,
            "green_energy": False,
            "lat": 50.469333,
            "lon": 3.865472,
        },
        "europe-west10": {
            "country_id": "DE",
            "city": "Berlin",
            "founding_year": 2023,
            "green_energy": False,
            # approximation based on city
            "lat": 52.5105672,
            "lon": 13.3806972,
        },
        "europe-west12": {
            "country_id": "IT",
            "city": "Turin",
            "founding_year": 2023,
            "green_energy": False,
            "lat": 45.146729,
            "lon": 7.742147,
        },
        "europe-west2": {
            "country_id": "GB",
            "city": "London",
            "founding_year": 2017,
            "green_energy": False,
            # approximation based on city
            "lat": 51.5090133,
            "lon": -0.2118157,
        },
        "europe-west3": {
            "country_id": "DE",
            "city": "Frankfurt",
            "founding_year": 2017,
            "green_energy": False,
            "lat": 50.12263,
            "lon": 8.974168,
        },
        "europe-west4": {
            "country_id": "NL",
            "city": "Eemshaven",
            "founding_year": 2018,
            "green_energy": False,
            "lat": 52.790105,
            "lon": 5.029219,
        },
        "europe-west6": {
            "country_id": "CH",
            "city": "Zurich",
            "founding_year": 2019,
            "green_energy": False,
            "lat": 47.445926,
            "lon": 8.210909,
        },
        "europe-west8": {
            "country_id": "IT",
            "city": "Milan",
            "founding_year": 2022,
            "green_energy": False,
            # approximation based on city
            "lat": 45.4615551,
            "lon": 9.1389572,
        },
        "europe-west9": {
            "country_id": "FR",
            "city": "Paris",
            "founding_year": 2022,
            "green_energy": False,
            # approximation based on city
            "lat": 48.8641797,
            "lon": 2.3109137,
        },
        "me-central1": {
            "country_id": "QA",
            "city": "Doha",
            "founding_year": 2023,
            "green_energy": False,
            # approximation based on city
            "lat": 25.272868,
            "lon": 51.4717522,
        },
        "me-central2": {
            "country_id": "SA",
            "city": "Dammam",
            "founding_year": 2023,
            "green_energy": False,
            # approximation based on city
            "lat": 26.3826288,
            "lon": 49.9675732,
        },
        "me-west1": {
            "country_id": "IL",
            "city": "Tel Aviv",
            "founding_year": 2022,
            "green_energy": False,
            # approximation based on city
            "lat": 32.0491183,
            "lon": 34.7891105,
        },
        "northamerica-northeast1": {
            "country_id": "CA",
            "city": "Montréal",
            "founding_year": 2018,
            "green_energy": True,
            # approximation based on city
            "lat": 45.4933996,
            "lon": -73.728239,
        },
        "northamerica-northeast2": {
            "country_id": "CA",
            "city": "Toronto",
            "founding_year": 2021,
            "green_energy": False,
            # approximation based on city
            "lat": 43.72666,
            "lon": -79.5355309,
        },
        "southamerica-east1": {
            "country_id": "BR",
            "city": "Osasco",
            "state": "São Paulo",
            "founding_year": 2017,
            "green_energy": False,
            # approximation based on city
            "lat": -23.5267431,
            "lon": -46.8096539,
        },
        "southamerica-west1": {
            "country_id": "CL",
            "city": "Santiago",
            "founding_year": 2021,
            "green_energy": False,
            "lat": -33.520515,
            "lon": -70.721695,
        },
        "us-central1": {
            "country_id": "US",
            "city": "Council Bluffs",
            "state": "Iowa",
            "founding_year": 2009,
            "green_energy": False,
            "lat": 41.168253,
            "lon": -95.796125,
        },
        "us-east1": {
            "country_id": "US",
            "city": "Moncks Corner",
            "state": "South Carolina",
            "founding_year": 2015,
            "green_energy": False,
            "lat": 33.064111,
            "lon": -80.043361,
        },
        "us-east4": {
            "country_id": "US",
            "city": "Ashburn",
            "state": "Virginia",
            "founding_year": 2017,
            "green_energy": False,
            "lat": 38.943331,
            "lon": -77.524336,
        },
        "us-east5": {
            "country_id": "US",
            "city": "Columbus",
            "state": "Ohio",
            "founding_year": 2022,
            "green_energy": False,
            # approximation based on city
            "lat": 39.9773124,
            "lon": -83.0423282,
        },
        "us-south1": {
            "country_id": "US",
            "city": "Dallas",
            "state": "Texas",
            "founding_year": 2022,
            "green_energy": False,
            "lat": 32.44317,
            "lon": -97.062324,
        },
        "us-west1": {
            "country_id": "US",
            "city": "The Dalles",
            "state": "Oregon",
            "founding_year": 2016,
            "green_energy": False,
            "lat": 45.632511,
            "lon": -121.202267,
        },
        "us-west2": {
            "country_id": "US",
            "city": "Los Angeles",
            "state": "California",
            "founding_year": 2018,
            "green_energy": False,
            # approximation based on city
            "lat": 34.0549694,
            "lon": -118.3753618,
        },
        "us-west3": {
            "country_id": "US",
            "city": "Salt Lake City",
            "state": "Utah",
            "founding_year": 2020,
            "green_energy": False,
            # approximation based on city
            "lat": 40.7386099,
            "lon": -111.9609998,
        },
        "us-west4": {
            "country_id": "US",
            "city": "Las Vegas",
            "state": "Nevada",
            "founding_year": 2020,
            "green_energy": False,
            "lat": 36.055625,
            "lon": -115.010226,
        },
    }

    # add API reference and display names
    for k, v in manual_data.items():
        v["api_reference"] = k
        if v.get("display_name") is None:
            v["display_name"] = v.get("city", v.get("state", ""))
            if v.get("display_name"):
                v["display_name"] = v["display_name"] + " (" + v["country_id"] + ")"
            else:
                v["display_name"] = v["country_id"]

    regions = _regions()
    items = []
    for region in regions:
        if region.name not in manual_data:
            raise KeyError(f"Unknown region metadata for {region.name}")
        item = {
            "vendor_id": vendor.vendor_id,
            "region_id": str(region.id),
            "name": region.name,
        }
        for k, v in manual_data[region.name].items():
            item[k] = v
        items.append(item)
    return items

inventory_zones

inventory_zones(vendor)

List all available GCP zones via API calls.

Source code in sc_crawler/vendors/gcp.py

def inventory_zones(vendor):
    """List all available GCP zones via API calls."""
    items = []
    regions = scmodels_to_dict(vendor.regions, keys=["name"])
    for zone in _zones():
        items.append(
            {
                "vendor_id": vendor.vendor_id,
                # example `zone.region`:
                # https://www.googleapis.com/compute/v1/projects/algebraic-pier-412621/regions/us-east4
                "region_id": regions[zone.region.split("/")[-1]].region_id,
                "zone_id": str(zone.id),
                "name": zone.name,
                "api_reference": zone.name,
                "display_name": zone.name,
            }
        )
    return items

inventory_servers

inventory_servers(vendor)

List all available GCP servers available in all zones.

Source code in sc_crawler/vendors/gcp.py

def inventory_servers(vendor):
    """List all available GCP servers available in all zones."""
    servers = parallel_fetch_servers(vendor, _search_servers, "name", "zones")
    servers = preprocess_servers(servers, vendor, add_vendor_id)
    return servers

inventory_server_prices

inventory_server_prices(vendor)

List all available GCP server ondemand prices in all regions.

Source code in sc_crawler/vendors/gcp.py

def inventory_server_prices(vendor):
    """List all available GCP server ondemand prices in all regions."""
    return _inventory_server_prices(vendor, Allocation.ONDEMAND)

inventory_server_prices_spot

inventory_server_prices_spot(vendor)

List all available GCP server spot prices in all regions.

Source code in sc_crawler/vendors/gcp.py

def inventory_server_prices_spot(vendor):
    """List all available GCP server spot prices in all regions."""
    return _inventory_server_prices(vendor, Allocation.SPOT)

inventory_storages

inventory_storages(vendor)

List all available GCP disk storage options available in all zones.

For more details on the disk types, check https://cloud.google.com/compute/docs/disks#disk-types.

Source code in sc_crawler/vendors/gcp.py

def inventory_storages(vendor):
    """List all available GCP disk storage options available in all zones.

    For more details on the disk types, check <https://cloud.google.com/compute/docs/disks#disk-types>."""
    vendor.progress_tracker.start_task(
        name="Scanning zone(s) for storage(s)", total=len(vendor.zones)
    )

    def search_storages(zone: Zone, vendor: Vendor) -> List[dict]:
        zone_storages = []
        for storage in _storages(zone.name):
            valid_sizes = storage.valid_disk_size.replace("GB", "").split("-")
            zone_storages.append(
                {
                    "storage_id": str(storage.id),
                    "vendor_id": vendor.vendor_id,
                    "name": storage.name,
                    "description": storage.description,
                    "storage_type": (
                        StorageType.SSD
                        if storage.name != "pd-standard"
                        else StorageType.HDD
                    ),
                    "max_iops": None,
                    "max_throughput": None,
                    "min_size": int(valid_sizes[0]),
                    "max_size": int(valid_sizes[1]),
                }
            )
        vendor.log(f"{len(zone_storages)} storage(s) found in {zone.name}.")
        vendor.progress_tracker.advance_task()
        return zone_storages

    with ThreadPoolExecutor(max_workers=8) as executor:
        storages = executor.map(search_storages, vendor.zones, repeat(vendor))
    storages = list(chain.from_iterable(storages))

    vendor.log(f"{len(storages)} storage(s) found in {len(vendor.zones)} zones.")
    storages = list({p["name"]: p for p in storages}.values())
    vendor.log(f"{len(storages)} unique storage(s) found.")
    storages = [s for s in storages if s["name"] in STORAGE_ALLOWLIST]
    vendor.log(f"{len(storages)} storage(s) after dropping items with complex pricing.")
    vendor.progress_tracker.hide_task()
    return storages

inventory_storage_prices

inventory_storage_prices(vendor)

List all available GCP disk storage prices in all regions.

Source code in sc_crawler/vendors/gcp.py

def inventory_storage_prices(vendor):
    """List all available GCP disk storage prices in all regions."""
    regions = scmodels_to_dict(vendor.regions, keys=["name"])
    skus = _skus_dict()
    items = []
    for storage in vendor.storages:
        storage_regions = skus["storage"][storage.name].keys()
        for storage_region in storage_regions:
            # skip edge regions
            region = regions.get(storage_region)
            if region is None:
                vendor.log(
                    f"Skip unknown '{storage_region}' region for {storage.name}",
                    DEBUG,
                )
                continue

            price, currency = skus["storage"][storage.name][storage_region]["ondemand"]
            for zone in region.zones:
                items.append(
                    {
                        "vendor_id": vendor.vendor_id,
                        "region_id": region.region_id,
                        "storage_id": storage.storage_id,
                        "unit": PriceUnit.GB_MONTH,
                        "price": price,
                        "currency": currency,
                    }
                )
    return items

inventory_traffic_prices

inventory_traffic_prices(vendor)

List inbound and outbound network traffic prices in all GCP regions.

Source code in sc_crawler/vendors/gcp.py

def inventory_traffic_prices(vendor):
    """List inbound and outbound network traffic prices in all GCP regions."""
    regions = scmodels_to_dict(vendor.regions, keys=["name"])
    skus = _skus("Compute Engine")
    items = []
    for sku in skus:
        # skip not processed items early
        if sku.category.resource_family != "Network":
            continue
        if sku.category.resource_group not in [
            "StandardInternetEgress",
            "StandardInternetIngress",
        ]:
            continue

        # helper variables
        traffic_regions = sku.service_regions
        tiered_rates = sku.pricing_info[0].pricing_expression.tiered_rates
        price_tiers = []
        for i in range(len(tiered_rates)):
            price_tiers.append(
                {
                    "lower": tiered_rates[i].start_usage_amount,
                    "upper": "Infinity"
                    if i == len(tiered_rates) - 1
                    else tiered_rates[i + 1].start_usage_amount,
                    "price": tiered_rates[i].unit_price.nanos / 1e9,
                }
            )

        for traffic_region in traffic_regions:
            region = regions.get(traffic_region)
            if region is None:
                vendor.log(
                    f"Skip unknown '{traffic_region}' region for {sku.description}",
                    DEBUG,
                )
                continue
            items.append(
                {
                    "vendor_id": vendor.vendor_id,
                    "region_id": region.region_id,
                    "price": max([t["price"] for t in price_tiers]),
                    "price_tiered": price_tiers,
                    "currency": tiered_rates[0].unit_price.currency_code,
                    "unit": PriceUnit.GB_MONTH,
                    "direction": (
                        TrafficDirection.OUT
                        if sku.category.resource_group == "StandardInternetEgress"
                        else TrafficDirection.IN
                    ),
                }
            )

    return items

inventory_ipv4_prices

inventory_ipv4_prices(vendor)

List the price of an attached IPv4 address in all GCP regions.

Note that this data was not found using the APIs (only unattached static IPs), so the values are recorded manually from https://cloud.google.com/vpc/network-pricing#ipaddress.

Source code in sc_crawler/vendors/gcp.py

def inventory_ipv4_prices(vendor):
    """List the price of an attached IPv4 address in all GCP regions.

    Note that this data was not found using the APIs (only unattached static IPs),
    so the values are recorded manually from <https://cloud.google.com/vpc/network-pricing#ipaddress>.
    """
    # skus = _skus("Compute Engine")
    # for sku in skus:
    #     if sku.category.resource_family != "Network":
    #         continue
    #     if sku.description == "Static Ip Charge":
    #         pass
    items = []
    for region in vendor.regions:
        items.append(
            {
                "vendor_id": vendor.vendor_id,
                "region_id": region.region_id,
                "price": 0.005,
                "currency": "USD",
                "unit": PriceUnit.HOUR,
            }
        )
    return items