feat(agent): Delivery Optimization geo endpoint as last-resort geolocation provider

When ipinfo.io (plus retry) and ifconfig.co both fail, the agent now asks
geo.prod.do.dsp.mp.microsoft.com/geo. It returns only CountryCode and
ExternalIpAddress, but it belongs to the DO/Windows Update endpoint set Autopilot
needs anyway, so it stays reachable where public geo services are blocked.
The session then gets a country-only device_location (source "do-geo", no
timezone, so timezone auto-set stays pending) instead of none.

- GeoLocationService: the three providers share one TryProvider skeleton;
  response parsers are internal and unit-tested; GeoLocationResult.Describe()
  skips missing parts ("DE" instead of ", , DE") in the event message and log.
- Failure trace carries doGeoError + lastResortProvider.
- Backend: geographic location keys skip empty region/city, so country-only
  sessions group as "DE" rather than ", DE"; drilldown still round-trips.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Monitoring/Transport/GeoLocationServiceTests.cs

@@ -0,0 +1,88 @@
+using AutopilotMonitor.Agent.V2.Core.Monitoring.Transport;
+using Newtonsoft.Json.Linq;
+using Xunit;
+
+namespace AutopilotMonitor.Agent.V2.Core.Tests.Monitoring.Transport
+{
+    /// <summary>
+    /// Response parsers of the geo providers. Addresses come from the RFC 5737 documentation
+    /// ranges — never real ones.
+    /// </summary>
+    public class GeoLocationServiceTests
+    {
+        [Fact]
+        public void ParseDoGeo_maps_country_and_egress_ip_and_ignores_the_do_service_fields()
+        {
+            var json = JObject.Parse(@"{
+                ""ExternalIpAddress"": ""203.0.113.7"",
+                ""CountryCode"": ""DE"",
+                ""KeyValue_EndpointFullUri"": ""https://kv601.prod.do.dsp.mp.microsoft.com/all"",
+                ""Version"": ""26A6D4D6994E8DA5"",
+                ""CacheId"": ""7"",
+                ""ContentCert"": false,
+                ""DownloadModeFailSafe"": """"
+            }");
+
+            var result = GeoLocationService.ParseDoGeo(json);
+
+            Assert.NotNull(result);
+            Assert.Equal("DE", result.Country);
+            Assert.Equal("203.0.113.7", result.Ip);
+            Assert.Equal("do-geo", result.Source);
+            Assert.Null(result.Region);
+            Assert.Null(result.City);
+            Assert.Null(result.Loc);
+            Assert.Null(result.Timezone);
+        }
+
+        [Theory]
+        [InlineData(@"{ ""ExternalIpAddress"": ""203.0.113.7"" }")]
+        [InlineData(@"{ ""ExternalIpAddress"": ""203.0.113.7"", ""CountryCode"": """" }")]
+        public void ParseDoGeo_without_country_is_a_failure(string body)
+        {
+            Assert.Null(GeoLocationService.ParseDoGeo(JObject.Parse(body)));
+        }
+
+        [Fact]
+        public void ParseIpInfo_maps_all_fields()
+        {
+            var json = JObject.Parse(@"{
+                ""ip"": ""198.51.100.4"", ""city"": ""Stuttgart"", ""region"": ""Baden-Wurttemberg"",
+                ""country"": ""DE"", ""loc"": ""48.7823,9.1770"", ""timezone"": ""Europe/Berlin""
+            }");
+
+            var result = GeoLocationService.ParseIpInfo(json);
+
+            Assert.Equal("DE", result.Country);
+            Assert.Equal("Baden-Wurttemberg", result.Region);
+            Assert.Equal("Stuttgart", result.City);
+            Assert.Equal("48.7823,9.1770", result.Loc);
+            Assert.Equal("Europe/Berlin", result.Timezone);
+            Assert.Equal("198.51.100.4", result.Ip);
+            Assert.Equal("ipinfo", result.Source);
+        }
+
+        [Fact]
+        public void ParseIfConfigCo_joins_coordinates_and_leaves_loc_empty_without_them()
+        {
+            var withCoords = GeoLocationService.ParseIfConfigCo(JObject.Parse(@"{
+                ""ip"": ""198.51.100.4"", ""country_iso"": ""DE"", ""region_name"": ""Bavaria"",
+                ""city"": ""Munich"", ""latitude"": 48.13, ""longitude"": 11.58, ""time_zone"": ""Europe/Berlin""
+            }"));
+            var withoutCoords = GeoLocationService.ParseIfConfigCo(JObject.Parse(@"{ ""country_iso"": ""DE"" }"));
+
+            Assert.Equal("48.13,11.58", withCoords.Loc);
+            Assert.Equal("DE", withCoords.Country);
+            Assert.Equal("Bavaria", withCoords.Region);
+            Assert.Equal("ifconfig.co", withCoords.Source);
+            Assert.Equal("", withoutCoords.Loc);
+        }
+
+        [Fact]
+        public void Describe_skips_the_parts_a_provider_did_not_return()
+        {
+            Assert.Equal("Stuttgart, BW, DE", new GeoLocationResult { City = "Stuttgart", Region = "BW", Country = "DE" }.Describe());
+            Assert.Equal("DE", new GeoLocationResult { Country = "DE", Region = "" }.Describe());
+        }
+    }
+}

src/Agent/AutopilotMonitor.Agent.V2.Core.Tests/Runtime/StartupEnvironmentProbesTests.cs

@@ -62,6 +62,18 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Runtime
             Assert.Equal("ipinfo.io", evt.Data["source"]);
         }
 
+        [Fact]
+        public void BuildGeoEvent_country_only_location_has_no_empty_parts_in_message()
+        {
+            var loc = new GeoLocationResult { Country = "DE", Source = "do-geo" };
+
+            var evt = StartupEnvironmentProbes.BuildGeoEvent(Config(), loc);
+
+            Assert.Equal("Device location: DE (via do-geo)", evt.Message);
+            Assert.Equal("DE", evt.Data["country"]);
+            Assert.Equal("do-geo", evt.Data["source"]);
+        }
+
         // ================================================================= Outbound IP
 
         [Fact]
@@ -98,6 +110,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Runtime
                 PrimaryError = "connection refused",
                 PrimaryRetryError = "DNS failure",
                 FallbackError = "timeout",
+                DoGeoError = "HTTP 403 (Forbidden)",
             };
 
             var evt = StartupEnvironmentProbes.BuildGeoFailureEvent(cfg, attempt);
@@ -108,6 +121,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Runtime
             Assert.Equal("connection refused", evt.Data["primaryError"]);
             Assert.Equal("DNS failure", evt.Data["primaryRetryError"]);
             Assert.Equal("timeout", evt.Data["fallbackError"]);
+            Assert.Equal("HTTP 403 (Forbidden)", evt.Data["doGeoError"]);
+            Assert.Equal("do-geo", evt.Data["lastResortProvider"]);
         }
 
         [Fact]
@@ -119,6 +134,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Tests.Runtime
             Assert.Equal("unknown", evt.Data["primaryError"]);
             Assert.Equal("unknown", evt.Data["primaryRetryError"]);
             Assert.Equal("unknown", evt.Data["fallbackError"]);
+            Assert.Equal("unknown", evt.Data["doGeoError"]);
         }
 
         // ================================================================= Timezone

src/Agent/AutopilotMonitor.Agent.V2.Core/Monitoring/Transport/GeoLocationService.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
@@ -25,6 +26,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
         /// </summary>
         public string Ip { get; set; }
 
+        /// <summary>"City, Region, Country" without the parts a provider did not return.</summary>
+        public string Describe() =>
+            string.Join(", ", new[] { City, Region, Country }.Where(p => !string.IsNullOrEmpty(p)));
+
         public Dictionary<string, object> ToDictionary()
         {
             return new Dictionary<string, object>
@@ -45,6 +50,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
         public string PrimaryError { get; set; }
         public string PrimaryRetryError { get; set; }
         public string FallbackError { get; set; }
+        public string DoGeoError { get; set; }
     }
 
     public static class GeoLocationService
@@ -52,6 +58,16 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
         private static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(5);
         private static readonly TimeSpan RetryDelay = TimeSpan.FromSeconds(2);
 
+        internal const string IpInfoUrl = "https://ipinfo.io/json";
+        internal const string IfConfigCoUrl = "https://ifconfig.co/json";
+
+        /// <summary>
+        /// Delivery Optimization's own geo endpoint. Country and egress IP only (no region, city,
+        /// coordinates or timezone), but it is part of the Windows Update / DO endpoint set that
+        /// Autopilot needs anyway, so it stays reachable in networks that block public geo services.
+        /// </summary>
+        internal const string DoGeoUrl = "https://geo.prod.do.dsp.mp.microsoft.com/geo";
+
         // Static HttpClient to avoid socket exhaustion from repeated short-lived instances.
         // HttpClient is designed to be reused across requests.
         private static readonly HttpClient SharedHttpClient = CreateSharedHttpClient();
@@ -68,7 +84,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
             var attempt = new GeoLocationAttemptResult();
 
             // Try ipinfo.io first
-            var (result, error) = await TryIpInfo(logger);
+            var (result, error) = await TryProvider("ipinfo.io", IpInfoUrl, ParseIpInfo, logger);
             if (result != null)
             {
                 attempt.Location = result;
@@ -80,7 +96,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
             logger?.Info($"GeoLocation: Retrying ipinfo.io after {RetryDelay.TotalSeconds}s...");
             await Task.Delay(RetryDelay);
 
-            (result, error) = await TryIpInfo(logger);
+            (result, error) = await TryProvider("ipinfo.io", IpInfoUrl, ParseIpInfo, logger);
             if (result != null)
             {
                 attempt.Location = result;
@@ -89,7 +105,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
             attempt.PrimaryRetryError = error;
 
             // Fallback to ifconfig.co
-            (result, error) = await TryIfConfigCo(logger);
+            (result, error) = await TryProvider("ifconfig.co", IfConfigCoUrl, ParseIfConfigCo, logger);
             if (result != null)
             {
                 attempt.Location = result;
@@ -97,108 +113,111 @@ namespace AutopilotMonitor.Agent.V2.Core.Monitoring.Transport
             }
             attempt.FallbackError = error;
 
+            // Last resort: Delivery Optimization geo — country only, but better than no location
+            (result, error) = await TryProvider("do-geo", DoGeoUrl, ParseDoGeo, logger);
+            if (result != null)
+            {
+                attempt.Location = result;
+                return attempt;
+            }
+            attempt.DoGeoError = error;
+
             logger?.Warning("GeoLocation: All providers failed, skipping location event");
             return attempt;
         }
 
-        private static async Task<(GeoLocationResult result, string error)> TryIpInfo(AgentLogger logger)
+        private static async Task<(GeoLocationResult result, string error)> TryProvider(
+            string name, string url, Func<JObject, GeoLocationResult> parse, AgentLogger logger)
         {
             try
             {
-                logger?.Info("GeoLocation: Querying ipinfo.io...");
+                logger?.Info($"GeoLocation: Querying {name}...");
 
-                using (var httpResponse = await SharedHttpClient.GetAsync("https://ipinfo.io/json"))
+                using (var httpResponse = await SharedHttpClient.GetAsync(url))
                 {
                     if (!httpResponse.IsSuccessStatusCode)
                     {
                         var error = $"HTTP {(int)httpResponse.StatusCode} ({httpResponse.ReasonPhrase})";
-                        logger?.Warning($"GeoLocation: ipinfo.io failed: {error}");
+                        logger?.Warning($"GeoLocation: {name} failed: {error}");
                         return (null, error);
                     }
 
                     var response = await httpResponse.Content.ReadAsStringAsync();
-                    var json = JObject.Parse(response);
-
-                    var result = new GeoLocationResult
+                    var result = parse(JObject.Parse(response));
+                    if (result == null)
                     {
-                        Country = json.Value<string>("country"),
-                        Region = json.Value<string>("region"),
-                        City = json.Value<string>("city"),
-                        Loc = json.Value<string>("loc"),
-                        Timezone = json.Value<string>("timezone"),
-                        Ip = json.Value<string>("ip"),
-                        Source = "ipinfo"
-                    };
+                        var error = "No country in response";
+                        logger?.Warning($"GeoLocation: {name} failed: {error}");
+                        return (null, error);
+                    }
 
-                    logger?.Info($"GeoLocation: ipinfo.io returned {result.City}, {result.Region}, {result.Country}");
+                    logger?.Info($"GeoLocation: {name} returned {result.Describe()}");
                     return (result, null);
                 }
             }
             catch (TaskCanceledException)
             {
-                var error = "Timeout (5s)";
-                logger?.Warning($"GeoLocation: ipinfo.io failed: {error}");
+                var error = $"Timeout ({RequestTimeout.TotalSeconds}s)";
+                logger?.Warning($"GeoLocation: {name} failed: {error}");
                 return (null, error);
             }
             catch (Exception ex)
             {
                 var error = ex.Message;
-                logger?.Warning($"GeoLocation: ipinfo.io failed: {error}");
+                logger?.Warning($"GeoLocation: {name} failed: {error}");
                 return (null, error);
             }
         }
 
-        private static async Task<(GeoLocationResult result, string error)> TryIfConfigCo(AgentLogger logger)
+        internal static GeoLocationResult ParseIpInfo(JObject json) =>
+            new GeoLocationResult
+            {
+                Country = json.Value<string>("country"),
+                Region = json.Value<string>("region"),
+                City = json.Value<string>("city"),
+                Loc = json.Value<string>("loc"),
+                Timezone = json.Value<string>("timezone"),
+                Ip = json.Value<string>("ip"),
+                Source = "ipinfo"
+            };
+
+        internal static GeoLocationResult ParseIfConfigCo(JObject json)
         {
-            try
+            var latitude = json.Value<string>("latitude") ?? "";
+            var longitude = json.Value<string>("longitude") ?? "";
+            var loc = !string.IsNullOrEmpty(latitude) && !string.IsNullOrEmpty(longitude)
+                ? $"{latitude},{longitude}"
+                : "";
+
+            return new GeoLocationResult
             {
-                logger?.Info("GeoLocation: Falling back to ifconfig.co...");
+                Country = json.Value<string>("country_iso"),
+                Region = json.Value<string>("region_name"),
+                City = json.Value<string>("city"),
+                Loc = loc,
+                Timezone = json.Value<string>("time_zone"),
+                Ip = json.Value<string>("ip"),
+                Source = "ifconfig.co"
+            };
+        }
 
-                using (var httpResponse = await SharedHttpClient.GetAsync("https://ifconfig.co/json"))
-                {
-                    if (!httpResponse.IsSuccessStatusCode)
-                    {
-                        var error = $"HTTP {(int)httpResponse.StatusCode} ({httpResponse.ReasonPhrase})";
-                        logger?.Warning($"GeoLocation: ifconfig.co failed: {error}");
-                        return (null, error);
-                    }
+        /// <summary>
+        /// The DO geo response carries DO service configuration next to the geo fields; only
+        /// <c>CountryCode</c> and <c>ExternalIpAddress</c> are location data. Without a country
+        /// the response is worthless as a location, so it counts as a failure (null).
+        /// </summary>
+        internal static GeoLocationResult ParseDoGeo(JObject json)
+        {
+            var country = json.Value<string>("CountryCode");
+            if (string.IsNullOrEmpty(country))
+                return null;
 
-                    var response = await httpResponse.Content.ReadAsStringAsync();
-                    var json = JObject.Parse(response);
-
-                    var latitude = json.Value<string>("latitude") ?? "";
-                    var longitude = json.Value<string>("longitude") ?? "";
-                    var loc = !string.IsNullOrEmpty(latitude) && !string.IsNullOrEmpty(longitude)
-                        ? $"{latitude},{longitude}"
-                        : "";
-
-                    var result = new GeoLocationResult
-                    {
-                        Country = json.Value<string>("country_iso"),
-                        Region = json.Value<string>("region_name"),
-                        City = json.Value<string>("city"),
-                        Loc = loc,
-                        Timezone = json.Value<string>("time_zone"),
-                        Ip = json.Value<string>("ip"),
-                        Source = "ifconfig.co"
-                    };
-
-                    logger?.Info($"GeoLocation: ifconfig.co returned {result.City}, {result.Region}, {result.Country}");
-                    return (result, null);
-                }
-            }
-            catch (TaskCanceledException)
+            return new GeoLocationResult
             {
-                var error = "Timeout (5s)";
-                logger?.Warning($"GeoLocation: ifconfig.co failed: {error}");
-                return (null, error);
-            }
-            catch (Exception ex)
-            {
-                var error = ex.Message;
-                logger?.Warning($"GeoLocation: ifconfig.co failed: {error}");
-                return (null, error);
-            }
+                Country = country,
+                Ip = json.Value<string>("ExternalIpAddress"),
+                Source = "do-geo"
+            };
         }
     }
 }

src/Agent/AutopilotMonitor.Agent.V2.Core/Runtime/StartupEnvironmentProbes.cs

@@ -18,7 +18,8 @@ namespace AutopilotMonitor.Agent.V2.Core.Runtime
     /// </para>
     /// <list type="bullet">
     ///   <item><b>device_location</b> (or <c>agent_trace</c> on failure) — IP-based geo via
-    ///     <see cref="GeoLocationService"/>. Skipped when <c>EnableGeoLocation=false</c>.</item>
+    ///     <see cref="GeoLocationService"/>. Skipped when <c>EnableGeoLocation=false</c>. The
+    ///     last-resort provider (DO geo) yields a country-only location without timezone.</item>
     ///   <item><b>timezone_auto_set</b> — only when <c>EnableTimezoneAutoSet=true</c> AND the
     ///     geo lookup returned an IANA timezone. Uses <see cref="TimezoneService"/> (tzutil).</item>
     ///   <item><b>ntp_time_check</b> — NTP offset from <c>NtpServer</c> (default time.windows.com).
@@ -225,7 +226,7 @@ namespace AutopilotMonitor.Agent.V2.Core.Runtime
                 Source = "StartupEnvironmentProbes",
                 Phase = EnrollmentPhase.Unknown,
                 Timestamp = DateTime.UtcNow,
-                Message = $"Device location: {location.City}, {location.Region}, {location.Country} (via {location.Source})",
+                Message = $"Device location: {location.Describe()} (via {location.Source})",
                 Data = location.ToDictionary(),
                 ImmediateUpload = true,
             };
@@ -266,8 +267,10 @@ namespace AutopilotMonitor.Agent.V2.Core.Runtime
                     { "primaryError", attempt?.PrimaryError ?? "unknown" },
                     { "primaryRetryError", attempt?.PrimaryRetryError ?? "unknown" },
                     { "fallbackError", attempt?.FallbackError ?? "unknown" },
+                    { "doGeoError", attempt?.DoGeoError ?? "unknown" },
                     { "primaryProvider", "ipinfo.io" },
                     { "fallbackProvider", "ifconfig.co" },
+                    { "lastResortProvider", "do-geo" },
                 },
             };
 

src/Backend/AutopilotMonitor.Functions.Tests/GeographicDrilldownFilterTests.cs

@@ -115,4 +115,21 @@ public class GeographicDrilldownFilterTests
         Assert.Equal("Saxony", fal.Region);
         Assert.Equal("Falkenstein", fal.City);
     }
+
+    [Theory]
+    [InlineData("city")]
+    [InlineData("region")]
+    [InlineData("country")]
+    public void CountryOnlySession_KeyIsTheCountryAndDrillsDownToItself(string groupBy)
+    {
+        // The agent's last-resort geo provider returns the country only.
+        var countryOnly = Geo("de-only", "DE", "", "");
+        var sessions = new List<SessionSummary> { countryOnly, Geo("de-bav", "DE", "Bavaria", "Munich") };
+
+        var key = GetGeographicMetricsFunction.GetLocationKey(countryOnly, groupBy);
+        var drilled = GetGeographicLocationSessionsFunction.FilterSessionsByLocation(sessions, key, groupBy);
+
+        Assert.Equal("DE", key); // never ", DE" or ", , DE"
+        Assert.Contains(drilled, s => s.SessionId == "de-only");
+    }
 }

src/Backend/AutopilotMonitor.Functions/Functions/Metrics/GetGeographicMetricsFunction.cs

@@ -301,11 +301,14 @@ namespace AutopilotMonitor.Functions.Functions.Metrics
             return groupBy.ToLower() switch
             {
                 "country" => session.GeoCountry,
-                "region" => $"{session.GeoRegion}, {session.GeoCountry}",
-                _ => !string.IsNullOrEmpty(session.GeoCity)
-                    ? $"{session.GeoCity}, {session.GeoRegion}, {session.GeoCountry}"
-                    : $"{session.GeoRegion}, {session.GeoCountry}"
+                "region" => JoinLocationParts(session.GeoRegion, session.GeoCountry),
+                _ => JoinLocationParts(session.GeoCity, session.GeoRegion, session.GeoCountry)
             };
         }
+
+        // Providers can omit region/city (the agent's last-resort provider returns the country
+        // only), so empty parts are skipped instead of producing keys like ", DE".
+        private static string JoinLocationParts(params string[] parts) =>
+            string.Join(", ", parts.Where(p => !string.IsNullOrEmpty(p)));
     }
 }