azure

sc_crawler.vendors.azure

SERVER_FEATURES module-attribute

SERVER_FEATURES = {'a': 'AMD processor', 'p': 'ARM processor', 'b': 'Block Storage performance', 'd': 'Local Disk', 'i': 'Isolated', 'l': 'Low Memory', 'm': 'Memory Intensive', 't': 'Tiny Memory', 's': 'Premium Storage capable', 'r': 'RDMA capable', 'e': 'Memory Optimized', 'x': 'Unmatched Memory Capacity'}

Map lowercase chars from the server name to features.

STORAGE_METER_MAPPING module-attribute

STORAGE_METER_MAPPING = {'P2 LRS Disk Mount': ('PremiumV2_LRS', 1), 'P1 LRS Disk': ('Premium_LRS', 4), 'P1 ZRS Disk': ('Premium_ZRS', 4), 'E1 LRS Disk': ('StandardSSD_LRS', 4), 'E1 ZRS Disk': ('StandardSSD_ZRS', 4), 'S4 LRS Disk': ('Standard_LRS', 32), 'Ultra LRS Provisioned Capacity': ('UltraSSD_LRS', 1)}

Map Storage price meter names to the Storage name and the related disk's size.

inventory_compliance_frameworks

inventory_compliance_frameworks(vendor)

Manual list of known compliance frameworks at Azure.

Data collected from https://learn.microsoft.com/en-us/azure/compliance/.

Source code in sc_crawler/vendors/azure.py

def inventory_compliance_frameworks(vendor):
    """Manual list of known compliance frameworks at Azure.

    Data collected from <https://learn.microsoft.com/en-us/azure/compliance/>.
    """
    return map_compliance_frameworks_to_vendor(
        vendor.vendor_id, ["hipaa", "soc2t2", "iso27001"]
    )

inventory_regions

inventory_regions(vendor)

List all regions via API call.

Location (country and state) and founding year were collected manually from https://datacenters.microsoft.com/globe/explore/ and its underlying JSON at https://datacenters.microsoft.com/globe/data/geo/regions.json.

City and the energy source information was collected from the sustainability fact sheets referenced in the above page and JSON.

Coordinates were provided by the Microsoft API, which doesn't seem to be very reliable.

Source code in sc_crawler/vendors/azure.py

def inventory_regions(vendor):
    """List all regions via API call.

    Location (country and state) and founding year
    were collected manually from
    <https://datacenters.microsoft.com/globe/explore/>
    and its underlying JSON at
    <https://datacenters.microsoft.com/globe/data/geo/regions.json>.

    City and the energy source information was collected from
    the sustainability fact sheets referenced in the above page and JSON.

    Coordinates were provided by the Microsoft API, which doesn't seem
    to be very reliable.
    """

    manual_datas = {
        # Canada
        "canadaeast": {
            "country_id": "CA",
            "state": "Quebec",
            "city": "Quebec City",
            "founding_year": 2016,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "canadacentral": {
            "country_id": "CA",
            "city": "Toronto",
            "founding_year": 2016,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        # United States
        "centralus": {
            "country_id": "US",
            "state": "Iowa",
            "founding_year": 2014,
            "green_energy": True,
        },
        "centraluseuap": {
            "country_id": "US",
            "state": "Iowa",
            "green_energy": True,
        },
        "eastus": {
            "country_id": "US",
            "city": "Boydton",
            "state": "Virginia",
            # official site says 2014 with a dead link, but it was 2012 as per
            # https://web.archive.org/web/20120530115120/http:/blogs.msdn.com/b/windowsazure/archive/2012/04/05/announcing-new-datacenter-options-for-windows-azure.aspx
            "founding_year": 2012,
            "green_energy": False,
        },
        "eastusstg": {
            "country_id": "US",
            "state": "Virginia",
            "green_energy": False,
        },
        "eastus2": {
            "country_id": "US",
            "city": "Boydton",
            "state": "Virginia",
            # official site says 2012 with a dead link, but it was 2014 as per
            # https://azure.microsoft.com/en-us/updates/general-availability-microsoft-azure-us-central-and-us-east-2-regions/
            "founding_year": 2014,
            "green_energy": False,
        },
        "eastus2euap": {
            "country_id": "US",
            "state": "Virginia",
            "green_energy": False,
        },
        "northcentralus": {
            "country_id": "US",
            "city": "Chicago",
            "state": "Illinois",
            "founding_year": 2009,
            "green_energy": False,
        },
        "southcentralus": {
            "country_id": "US",
            "state": "Texas",
            "city": "San Antonio",
            "founding_year": 2008,
            "green_energy": True,
        },
        "southcentralusstg": {
            "country_id": "US",
            "state": "Texas",
            "city": "San Antonio",
        },
        "westcentralus": {
            "country_id": "US",
            "state": "Wyoming",
            "city": "Cheyenne",
            "founding_year": 2016,
            "green_energy": False,
        },
        "westus": {
            "country_id": "US",
            "state": "California",
            "founding_year": 2012,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "westus2": {
            "country_id": "US",
            "state": "Washington",
            "founding_year": 2007,
            "green_energy": False,
        },
        "westus3": {
            "country_id": "US",
            "state": "Arizona",
            "city": "Phoenix",
            "founding_year": 2021,
            "green_energy": False,
        },
        # Mexico
        "mexicocentral": {
            "country_id": "ZA",
            "state": "Querétaro",
            "founding_year": 2024,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        # South America
        "brazilsouth": {
            "country_id": "BR",
            "state": "Campinas",
            "founding_year": 2014,
            "green_energy": False,
        },
        "brazilsoutheast": {
            "country_id": "US",
            "city": "Rio de Janeiro",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        # not production region?
        # https://github.com/Azure/azure-dev/issues/2165#issuecomment-1542948509
        "brazilus": {
            "country_id": "BR",
        },
        # Asia Pacific
        "australiacentral": {
            "country_id": "AU",
            "city": "Canberra",
            "founding_year": 2018,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "australiacentral2": {
            "country_id": "AU",
            "city": "Canberra",
            "founding_year": 2018,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "australiaeast": {
            "country_id": "AU",
            "city": "Sydney",
            "state": "New South Wales",
            "founding_year": 2014,
            "green_energy": False,
        },
        "australiasoutheast": {
            "country_id": "AU",
            "city": "Melbourne",
            "state": "Victoria",
            "founding_year": 2014,
            "green_energy": False,
        },
        "eastasia": {
            "country_id": "HK",
            "founding_year": 2010,
            "green_energy": False,
        },
        "southeastasia": {
            "country_id": "SG",
            "city": "Singapore",
            "founding_year": 2010,
            "green_energy": False,
        },
        "japaneast": {
            "country_id": "JP",
            "city": "Tokyo",
            "founding_year": 2014,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "japanwest": {
            "country_id": "JP",
            "city": "Osaka",
            "founding_year": 2014,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "jioindiacentral": {
            "country_id": "IN",
            "city": "Nagpur",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "jioindiawest": {
            "country_id": "IN",
            "city": "Jamnagar",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "centralindia": {
            "country_id": "IN",
            "state": "Pune",
            "founding_year": 2015,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "southindia": {
            "country_id": "IN",
            "state": "Chennai",
            "founding_year": 2015,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "westindia": {
            "country_id": "IN",
            "state": "Mumbai",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "koreacentral": {
            "country_id": "KR",
            "city": "Seoul",
            "founding_year": 2017,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "koreasouth": {
            "country_id": "KR",
            "city": "Busan",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "newzealandnorth": {
            "country_id": "NZ",
            "city": "Auckland",
            "founding_year": 2024,
            "green_energy": False,
        },
        # Europe
        "francecentral": {
            "country_id": "FR",
            "city": "Paris",
            "founding_year": 2018,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "francesouth": {
            "country_id": "FR",
            "city": "Marseille",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "germanynorth": {
            "country_id": "DE",
            "city": "Berlin",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "germanywestcentral": {
            "country_id": "DE",
            "city": "Frankfurt",
            "founding_year": 2019,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "italynorth": {
            "country_id": "IT",
            "city": "Milan",
            "founding_year": 2023,
            "green_energy": False,
        },
        "northeurope": {
            "country_id": "IE",
            "city": "Dublin",
            "founding_year": 2009,
            "green_energy": False,
        },
        "norwayeast": {
            "country_id": "NO",
            "city": "Oslo",
            "founding_year": 2019,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "norwaywest": {
            "country_id": "NO",
        },
        "polandcentral": {
            "country_id": "PL",
            "city": "Warsaw",
            "founding_year": 2023,
            "green_energy": False,
        },
        "spaincentral": {
            "country_id": "ES",
            "city": "Madrid",
            "founding_year": 2024,
            "green_energy": False,
        },
        "swedencentral": {
            "country_id": "SE",
            "city": "Gävle and Sandviken",
            "founding_year": 2021,
            "green_energy": False,
        },
        "switzerlandnorth": {
            "country_id": "CH",
            "city": "Zürich",
            "founding_year": 2019,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "switzerlandwest": {
            "country_id": "CH",
            "city": "Geneva",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "uksouth": {
            "country_id": "GB",
            "city": "London",
            "founding_year": 2016,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "ukwest": {
            "country_id": "GB",
            "city": "Cardiff",
            "founding_year": 2017,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "westeurope": {
            "country_id": "NL",
            "founding_year": 2010,
            "green_energy": False,
        },
        # Middle East
        "israelcentral": {
            "country_id": "IL",
            "founding_year": 2023,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "qatarcentral": {
            "country_id": "QA",
            "city": "Doha",
            "founding_year": 2022,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "uaecentral": {
            "country_id": "AE",
            "city": "Abu Dhabi",
        },
        "uaenorth": {
            "country_id": "AE",
            "city": "Dubai",
            "founding_year": 2019,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        # Africa
        "southafricanorth": {
            "country_id": "ZA",
            "city": "Johannesburg",
            "founding_year": 2019,
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        "southafricawest": {
            "country_id": "ZA",
            "city": "Cape Town",
            # unknown as no sustainability fact sheet found
            "green_energy": False,
        },
        # China TODO enable
    }

    items = []
    for region in _regions():
        if region["metadata"]["region_type"] != "Physical":
            continue
        # no idea what are these
        if region["name"].endswith("stg"):
            continue
        # not production region?
        # https://github.com/Azure/azure-dev/issues/2165#issuecomment-1542948509
        if region["name"] == "brazilus":
            continue
        # exclude for now as this new region is popping up and being removed
        # from their API response randomly, so messing with git history
        if region["name"] == "newzealandnorth":
            continue
        manual_data = manual_datas.get(region["name"])
        if not manual_data:
            raise KeyError(f"No manual data found for {region['name']}.")
        items.append(
            {
                "vendor_id": vendor.vendor_id,
                "region_id": region["name"],
                "name": region["display_name"],
                "api_reference": region["name"],
                "display_name": (
                    region["display_name"] + " (" + manual_data["country_id"] + ")"
                ),
                "country_id": manual_data["country_id"],
                "state": manual_data.get("state"),
                "city": manual_data.get("city"),
                "address_line": None,
                "zip_code": None,
                "lat": region["metadata"]["latitude"],
                "lon": region["metadata"]["longitude"],
                "founding_year": manual_data.get("founding_year"),
                "green_energy": manual_data.get("green_energy"),
            }
        )
    return items

inventory_zones

inventory_zones(vendor)

List all availability zones.

API call to list existing availability zones ("1", "2", and "3") for each region, and creating a dummy "0" zone for the regions without availability zones.

Source code in sc_crawler/vendors/azure.py

def inventory_zones(vendor):
    """List all availability zones.

    API call to list existing availability zones ("1", "2", and "3")
    for each region, and creating a dummy "0" zone for the regions
    without availability zones.
    """
    items = []
    resources = _resources("Microsoft.Compute")
    locations = [i for i in resources if i["resource_type"] == "virtualMachines"][0]
    locations = {item["location"]: item["zones"] for item in locations["zone_mappings"]}
    for region in vendor.regions:
        # default to zone with 0 ID if there are no real availability zones
        region_zones = locations.get(region.name, ["0"])
        for zone in region_zones:
            items.append(
                {
                    "vendor_id": vendor.vendor_id,
                    "region_id": region.region_id,
                    "zone_id": zone,
                    "name": zone,
                    "api_reference": zone,
                    "display_name": region.region_id + "-" + zone,
                }
            )
    return items

inventory_servers

inventory_servers(vendor)

List all available instance types in all regions.

Source code in sc_crawler/vendors/azure.py

def inventory_servers(vendor):
    """List all available instance types in all regions."""
    servers = _servers()
    for i in range(len(servers) - 1, -1, -1):
        name = servers[i].get("name")
        # drop Basic servers as to be deprecated by Aug 2024
        if name.startswith("Basic"):
            vendor.log(f"Excluding deprecated: {name}")
            servers.pop(i)
        # servers that are likely to be not available, with zero pricing
        if name.endswith("Promo"):
            vendor.log(f"Excluding nonsense pricing: {name}")
            servers.pop(i)
        # servers probably not intended for our eyes
        if "Internal" in name:
            vendor.log(f"Excluding internal server: {name}")
            servers.pop(i)
        # servers randomly switching between active/inactive status
        # TODO review from time to time
        if name in ["Standard_M896ixds_32_v3", "Standard_M64-32bds_1_v3"]:
            vendor.log(f"Excluding server with questionable availability: {name}")
            servers.pop(i)
    servers = preprocess_servers(servers, vendor, _standardize_server)
    return servers

inventory_server_prices

inventory_server_prices(vendor)

List all known server ondemand prices in all regions using the Azure Retail Pricing API.

More information: https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices.

Source code in sc_crawler/vendors/azure.py

def inventory_server_prices(vendor):
    """List all known server ondemand prices in all regions using the Azure Retail Pricing API.

    More information: <https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices>.
    """
    return _inventory_server_prices(vendor, Allocation.ONDEMAND)

inventory_server_prices_spot

inventory_server_prices_spot(vendor)

List all known server spot prices in all regions using the Azure Retail Pricing API.

See details at inventory_server_prices.

Source code in sc_crawler/vendors/azure.py

def inventory_server_prices_spot(vendor):
    """List all known server spot prices in all regions using the Azure Retail Pricing API.

    See details at [inventory_server_prices][sc_crawler.vendors.azure.inventory_server_prices].
    """
    return _inventory_server_prices(vendor, Allocation.SPOT)

inventory_storages

inventory_storages(vendor)

List all storage options via the Compute resource manager client.

For more information, see https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types.

Source code in sc_crawler/vendors/azure.py

def inventory_storages(vendor):
    """List all storage options via the Compute resource manager client.

    For more information, see <https://learn.microsoft.com/en-us/azure/virtual-machines/disks-types>.
    """
    vendor.progress_tracker.start_task(
        name="Fetching list of compute resources", total=None
    )

    disks = []
    for resource in _compute_resources():
        if resource["resource_type"] == "disks":
            disks.append(resource)

    disks = list({d["name"]: d for d in disks}.values())
    vendor.progress_tracker.hide_task()

    items = []
    for disk in disks:

        def _search(values):
            return list_search(disk["capabilities"], "name", values)["value"]

        storage_type = (
            StorageType.HDD
            if "Standard" in disk["name"] and "SSD" not in disk["name"]
            else StorageType.SSD
        )
        redundancy_type = (
            "Locally Redundant Storage"
            if "LRS" in disk["name"]
            else "Zone-Redundant Storage"
        )
        description = f"{disk['tier']} tier {storage_type.name} ({redundancy_type})"

        items.append(
            {
                "storage_id": disk["name"],
                "vendor_id": vendor.vendor_id,
                "name": disk["name"],
                "description": description,
                "storage_type": storage_type,
                "max_iops": _search(["MaxIOpsReadWrite", "MaxIOps"]),
                "max_throughput": _search(
                    ["MaxBandwidthMBpsReadWrite", "MaxBandwidthMBps"]
                ),
                # NOTE this is 16TB for most drives?!
                "min_size": _search("MinSizeGiB"),
                "max_size": _search("MaxSizeGiB"),
            }
        )
    return items

inventory_storage_prices

inventory_storage_prices(vendor)

Look up Storage prices via the Azure Retail Prices API.

For more information, see https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices.

Source code in sc_crawler/vendors/azure.py

def inventory_storage_prices(vendor):
    """Look up Storage prices via the Azure Retail Prices API.

    For more information, see <https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices>.
    """
    vendor.progress_tracker.start_task(
        name="Fetching list of storage resources", total=None
    )
    retail_prices = _prices("$filter=serviceName eq 'Storage'")
    vendor.progress_tracker.hide_task()

    regions = scmodels_to_dict(vendor.regions, keys=["region_id"])
    storages = scmodels_to_dict(vendor.storages, keys=["storage_id"])

    items = []
    for p in retail_prices:
        mapping = STORAGE_METER_MAPPING.get(p["meterName"])
        if (
            mapping
            and mapping[0] in storages.keys()
            and p["armRegionName"] in regions.keys()
        ):
            items.append(
                {
                    "vendor_id": vendor.vendor_id,
                    "region_id": p["armRegionName"],
                    "storage_id": mapping[0],
                    "unit": PriceUnit.GB_MONTH,
                    "price": p["retailPrice"] / mapping[1],
                    "currency": p["currencyCode"],
                }
            )
    return items

inventory_traffic_prices

inventory_traffic_prices(vendor)

Look up Internet Egress/Ingress prices via the Azure Retail Prices API.

For more information, see https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices.

Source code in sc_crawler/vendors/azure.py

def inventory_traffic_prices(vendor):
    """Look up Internet Egress/Ingress prices via the Azure Retail Prices API.

    For more information, see <https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices>.
    """

    def get_tiers(prices: List[dict]) -> List[dict]:
        def prep_tiers(d: dict) -> dict:
            return {
                "lower": d.get("tierMinimumUnits", 0),
                "price": d["retailPrice"],
            }

        tiers = [prep_tiers(p) for p in prices]
        tiers.sort(key=lambda x: x.get("lower"))
        for i in range(len(tiers)):
            if i == len(tiers) - 1:
                tiers[i]["upper"] = "Infinity"
            else:
                tiers[i]["upper"] = tiers[i + 1]["lower"]
        return tiers

    def by_region(prices: List[dict], region: str) -> List[dict]:
        return [p for p in prices if p["armRegionName"] == region]

    vendor.progress_tracker.start_task(
        name="Fetching list of traffic prices", total=None
    )
    inbound_prices = _prices(
        "$filter=serviceFamily eq 'Networking' and meterName eq 'Standard Data Transfer In'"
    )
    outbound_prices = _prices(
        "$filter=serviceFamily eq 'Networking' and "
        "meterName eq 'Standard Data Transfer Out' and "
        "productName eq 'Bandwidth - Routing Preference: Internet'"
    )
    vendor.progress_tracker.hide_task()

    items = []
    regions = scmodels_to_dict(vendor.regions, keys=["api_reference"])
    for region in regions.values():
        for direction in ["inbound", "outbound"]:
            prices = inbound_prices if direction == "inbound" else outbound_prices
            tiers = get_tiers(by_region(prices, region.api_reference))
            if tiers:
                items.append(
                    {
                        "vendor_id": vendor.vendor_id,
                        "region_id": region.region_id,
                        "price": max([t["price"] for t in tiers]),
                        "price_tiered": tiers,
                        "currency": prices[0].get("currencyCode", "USD"),
                        "unit": PriceUnit.GB_MONTH,
                        "direction": (
                            TrafficDirection.IN
                            if direction == "inbound"
                            else TrafficDirection.OUT
                        ),
                    }
                )
    return items

inventory_ipv4_prices

inventory_ipv4_prices(vendor)

Look up Internet Egress/Ingress prices via the Azure Retail Prices API.

For more information, see https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices.

Source code in sc_crawler/vendors/azure.py

def inventory_ipv4_prices(vendor):
    """Look up Internet Egress/Ingress prices via the Azure Retail Prices API.

    For more information, see <https://learn.microsoft.com/en-us/rest/api/cost-management/retail-prices/azure-retail-prices>.
    """

    vendor.progress_tracker.start_task(
        name="Fetching list of traffic prices", total=None
    )
    prices = _prices(
        "$filter=serviceFamily eq 'Networking' and "
        "meterName eq 'Basic IPv4 Dynamic Public IP' and "
        "type eq 'Consumption'"
    )
    vendor.progress_tracker.hide_task()

    items = []
    regions = scmodels_to_dict(vendor.regions, keys=["api_reference"])
    for region in regions.values():
        price = list_search(prices, "armRegionName", region.api_reference)
        if price:
            items.append(
                {
                    "vendor_id": vendor.vendor_id,
                    "region_id": region.region_id,
                    "price": price["retailPrice"],
                    "currency": price.get("currencyCode", "USD"),
                    "unit": PriceUnit.HOUR,
                }
            )
    return items