Import from e4b104792206ee7ea64bf39c6b7d2c0c230f9d14

1 files changed, 296 insertions, 0 deletions

diff --git a/server/src/datex2.cppm b/server/src/datex2.cppm
new file mode 100644
index 0000000..b507e6f
--- /dev/null
+++ b/server/src/datex2.cppm
@@ -0,0 +1,296 @@
+module;
+
+#include <boost/geometry/algorithms/is_empty.hpp>
+#include <boost/geometry/srs/transformation.hpp>
+#include <boost/geometry/srs/epsg.hpp>
+
+#include <pugixml.hpp>
+
+export module routemon:datex2;
+
+import std;
+import :geo;
+import :time;
+import :util;
+
+using namespace std::literals::string_view_literals;
+
+namespace routemon::datex2 {
+
+  export struct situation;
+
+  export struct road_closure {
+    std::weak_ptr<situation> parent;
+    std::optional<time::period_seq> validity;
+    std::vector<geo::point> relevant_points = {};
+    std::vector<std::shared_ptr<geo::linestring>> relevant_line_strings = {};
+  };
+
+  export struct situation {
+    std::string id;
+    std::optional<geo::point> location = std::nullopt;  // as shown on the map, not used for querying
+    std::vector<std::string> comments = {};
+    std::vector<std::shared_ptr<road_closure>> road_closures = {};
+  };
+
+  export struct situation_publication {
+    time::timestamp publication_time;
+    std::vector<std::shared_ptr<situation>> situations;
+  };
+
+  auto parse_timestamp(char const* in) -> std::optional<time::timestamp> {
+    auto res = time::timestamp{};
+    auto is = std::istringstream{in};
+    is >> std::chrono::parse("%Y-%m-%dT%H:%M:%SZ", res);
+    return is.fail() ? std::nullopt : std::make_optional(res);
+  }
+
+  export class loader {
+    // ETRS 89 (EPSG:4258) -> WGS 84 (EPSG:4326)
+    bgeo::srs::transformation<bgeo::srs::static_epsg<4258>, bgeo::srs::static_epsg<4326>> etrs89_to_wgs84_{};
+
+    std::multiset<std::string> warnings_;
+
+    auto add_location_from_xml(road_closure& rc, pugi::xml_node const& loc_xml) -> void {
+      auto loc_xml_type = std::string_view{loc_xml.attribute("xsi:type").value()};
+      if (loc_xml_type == "loc:ItineraryByIndexedLocations") {
+        for (auto const loc_cont_xml : loc_xml.children("loc:locationContainedInItinerary")) {
+          add_location_from_xml(rc, loc_cont_xml.child("loc:location"));
+        }
+      } else if (loc_xml_type == "loc:LinearLocation" || loc_xml_type == "loc:SingleRoadLinearLocation") {
+        auto const& loc_gml_xml = loc_xml.child("loc:gmlLineString");
+        if (!loc_gml_xml)
+          return;
+
+        auto const srs_name = std::string_view{loc_gml_xml.attribute("srsName").value()};
+        if (srs_name != "WGS 84"sv) {
+          warnings_.insert(std::format("don't now how to handle the CRS {}", srs_name));
+          return;
+        }
+        auto const pos_list_str = std::string_view{loc_gml_xml.child_value("loc:posList")};
+        // lat1 long1 lat2 long2 ... lat(n-1) long(n-1) latn longn
+
+        auto ls = std::make_shared<geo::linestring>();
+
+        auto lat_set = false;
+        auto lat = 0.0;
+        for (auto const lat_or_long_str : std::views::split(pos_list_str, " "sv)) {
+          auto mlat_or_long = util::parse_double(std::string_view{lat_or_long_str});
+          if (!mlat_or_long) {
+            warnings_.insert(std::format("failed to parse coordinate {:?}", std::string_view{lat_or_long_str}));
+            return;
+          }
+
+          if (!lat_set) {
+            lat = *mlat_or_long;
+            lat_set = true;
+          } else {
+            bgeo::append(*ls, geo::point{*mlat_or_long, lat});
+            lat = 0;
+            lat_set = false;
+          }
+        }
+
+        if (bgeo::is_empty(*ls)) {
+          warnings_.emplace("empty line string in data set");
+          return;
+        }
+
+        rc.relevant_line_strings.push_back(ls);
+      } else if (loc_xml_type == "loc:PointLocation") {
+        auto const& coords_xml = loc_xml.child("loc:pointByCoordinates").child("loc:pointCoordinates");
+        if (!coords_xml)
+          return;
+
+        auto mlat = util::parse_double(coords_xml.child_value("loc:latitude"));
+        auto mlon = util::parse_double(coords_xml.child_value("loc:longitude"));
+        if (!mlat || !mlon) {
+          warnings_.emplace("failed to parse PointLocation coordinates");
+          return;
+        }
+
+        // Vaag genoeg zegt NDW dat het hier om WGS 84 gaat:
+        //   https://docs.ndw.nu/en/dataformaten/datex2-v3/elementen/locationreferencing/pointCoordinates/
+        // maar heeft het UML-model van DATEX II v3 het over ETRS 89:
+        //   https://docs.datex2.eu/_static/data/v3.7/umlmodel/html/EARoot/EA3/EA3/EA5/EA676.htm
+
+        auto const coords_etrs89 = geo::point{*mlon, *mlat};
+        auto coords_wgs84 = geo::point{};
+        etrs89_to_wgs84_.forward(coords_etrs89, coords_wgs84);
+
+        rc.relevant_points.push_back(coords_wgs84);
+      } else {
+        warnings_.insert(std::format("don't know how to hande location of type {}, ignoring", loc_xml.attribute("xsi:type").value()));
+        return;
+      }
+    }
+
+    auto handle_road_or_carriageway_or_lane_management(pugi::xml_node const& record_xml, std::weak_ptr<situation> parent) -> std::optional<std::shared_ptr<road_closure>> {
+      auto const type = std::string_view{record_xml.child("sit:roadOrCarriagewayOrLaneManagementType").child_value()};
+      if (type != "carriagewayClosures" && type != "roadClosed")
+        // TODO: checken of er nog andere types fietsers de doorgang zouden kunnen blokkeren?
+        return std::nullopt;
+
+      auto const& restricted_vehicle_types_xml = record_xml.child("sit:forVehiclesWithCharacteristicsOf");
+      bool likely_restriction_for_bikes = restricted_vehicle_types_xml.empty();
+      for (auto const vehicle_type_xml : restricted_vehicle_types_xml.children("com:vehicleType")) {
+        auto vehicle_type = std::string_view{vehicle_type_xml.child_value()};
+        if (vehicle_type == "anyVehicle" || vehicle_type == "bicycle" ||
+            vehicle_type == "unknown" || vehicle_type == "other") {
+          likely_restriction_for_bikes = true;
+        }
+      }
+      if (!likely_restriction_for_bikes)
+        return std::nullopt;
+
+      //---- Check if within the defined validity period
+
+      auto validity = std::optional<time::period_seq>{};
+      auto const& validity_xml = record_xml.child("sit:validity");
+      if (validity_xml && validity_xml.child_value("com:validityStatus") == "definedByValidityTimeSpec"sv) {
+        auto const& validity_spec_xml = validity_xml.child("com:validityTimeSpecification");
+
+        auto valid_periods = std::vector<time::period>{};
+        auto exception_periods = std::vector<time::period>{};
+
+        // TODO: com:overallEndTime may be missing (according to the DATEX II v3 data model)
+        auto const overall_start_time = parse_timestamp(validity_spec_xml.child_value("com:overallStartTime"));
+        auto const overall_end_time   = parse_timestamp(validity_spec_xml.child_value("com:overallEndTime"));
+        if (overall_start_time && overall_end_time && *overall_start_time < *overall_end_time) {
+          valid_periods.emplace_back(*overall_start_time, *overall_end_time);
+
+          for (auto const valid_period_xml : validity_xml.children("com:validPeriod")) {
+            auto const start_of_period = parse_timestamp(valid_period_xml.child_value("com:startOfPeriod"));
+            auto const end_of_period   = parse_timestamp(valid_period_xml.child_value("com:endOfPeriod"));
+            if (start_of_period && end_of_period && *start_of_period < *end_of_period) {
+              valid_periods.emplace_back(*start_of_period, *end_of_period);
+            }
+          }
+          for (auto const exception_period_xml : validity_xml.children("com:exceptionPeriod")) {
+            auto const start_of_period = parse_timestamp(exception_period_xml.child_value("com:startOfPeriod"));
+            auto const end_of_period   = parse_timestamp(exception_period_xml.child_value("com:endOfPeriod"));
+            if (start_of_period && end_of_period && *start_of_period < *end_of_period) {
+              exception_periods.emplace_back(*start_of_period, *end_of_period);
+            }
+          }
+
+          validity = time::period_seq{valid_periods.begin(), valid_periods.end()}
+            .except(time::period_seq{exception_periods.begin(), exception_periods.end()});
+        } else {
+          warnings_.insert(std::format("invalid overall start / end time (start time: {}, end time: {})",
+                                        validity_spec_xml.child_value("com:overallStartTime"),
+                                        validity_spec_xml.child_value("com:overallEndTime")));
+          return std::nullopt;
+        }
+      }
+
+      //---- Try to extract the location info
+
+      auto rc = std::make_shared<road_closure>(std::move(parent), validity);
+      add_location_from_xml(*rc, record_xml.child("sit:locationReference"));
+      return rc;
+    }
+
+  public:
+    [[nodiscard]] auto load_situation_publication(std::string const& filename) -> situation_publication {
+      auto doc = pugi::xml_document{};
+      if (auto result = doc.load_file(filename.c_str()); !result) {
+        throw std::runtime_error{result.description()};
+      }
+      auto payload_xml = doc.child("mc:messageContainer").child("mc:payload");
+      auto mpublication_time = parse_timestamp(payload_xml.child_value("com:publicationTime"));
+      if (!mpublication_time)
+        throw std::runtime_error{"provided publication does not name publication time"};
+
+      auto situations = std::vector<std::shared_ptr<situation>>{};
+      for (auto const sit_xml : payload_xml.children("sit:situation")) {
+        auto id = std::string_view{sit_xml.attribute("id").value()};
+
+        auto const sit = std::make_shared<situation>(std::string{id});
+        situations.push_back(sit);
+
+        auto const& header_info_xml = sit_xml.child("sit:headerInformation");
+        if (header_info_xml.child_value("com:informationStatus") != "real"sv)
+          continue;
+
+        for (auto const record_xml : sit_xml.children("sit:situationRecord")) {
+          auto const record_type = std::string_view{record_xml.attribute("xsi:type").value()};
+          auto const primary_record_types = std::unordered_set<std::string_view>{
+            "sit:Roadworks",
+            /* { */ "sit:MaintenanceWorks",
+            /* | */ "sit:ConstructionWorks",
+            /* } */
+            "sit:Obstruction",
+            /* { */ "sit:EnvironmentalObstruction",
+            /* | */ "sit:GeneralObstruction",
+            /* | */ "sit:InfrastructureDamageObstruction",
+            /* } */
+            "sit:Activity",
+            /* { */ "sit:PublicEvent",
+            /* } */
+          };
+
+          if (record_type == "sit:RoadOrCarriagewayOrLaneManagement") {
+            if (auto rc = handle_road_or_carriageway_or_lane_management(record_xml, sit)) {
+              sit->road_closures.push_back(*rc);
+            }
+          } else if (primary_record_types.contains(record_type)) {
+            for (auto const comment_xml : record_xml.children("sit:generalPublicComment")) {
+//            if (comment_xml.child_value("sit:commentType") == "internalNote"sv) {
+                auto candidate = std::optional<std::pair<std::string_view, std::string_view>>{}; // (text, language)
+                for (auto const comment_value_xml : comment_xml.child("sit:comment").child("com:values").children("com:value")) {
+                  if (!candidate ||
+                      comment_value_xml.attribute("lang").value() == "nl"sv ||
+                      (candidate->second != "nl"sv && comment_value_xml.attribute("lang").value() == "nl"sv)) {
+                    candidate = std::make_pair(comment_value_xml.child_value(), comment_value_xml.attribute("lang").value());
+                  }
+                }
+                if (candidate) {
+                  auto already_present = false;
+                  for (auto const& comment : sit->comments)
+                    already_present = already_present || comment == candidate->first;
+                  if (!already_present) {
+                    sit->comments.emplace_back(candidate->first);
+                  }
+                }
+//            }
+            }
+
+            if (auto const location_ref_xml = record_xml.child("sit:locationReference")) {
+              if (location_ref_xml.attribute("xsi:type").value() == "loc:PointLocation"sv) {
+                if (auto const coords_xml = location_ref_xml.child("loc:pointByCoordinates").child("loc:pointCoordinates")) {
+                  auto const mlat = util::parse_double(coords_xml.child_value("loc:latitude"));
+                  auto const mlon = util::parse_double(coords_xml.child_value("loc:longitude"));
+                  if (mlat && mlon) {
+                    // Vaag genoeg zegt NDW dat het hier om WGS 84 gaat:
+                    //   https://docs.ndw.nu/en/dataformaten/datex2-v3/elementen/locationreferencing/pointCoordinates/
+                    // maar heeft het UML-model van DATEX II v3 het over ETRS 89:
+                    //   https://docs.datex2.eu/_static/data/v3.7/umlmodel/html/EARoot/EA3/EA3/EA5/EA676.htm
+
+                    auto const coords_etrs89 = geo::point{*mlon, *mlat};
+                    auto coords_wgs84 = geo::point{};
+                    etrs89_to_wgs84_.forward(coords_etrs89, coords_wgs84);
+
+                    if (!sit->location) {
+                      sit->location = coords_wgs84;
+                    }
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+
+      return {
+        .publication_time = *mpublication_time,
+        .situations       = situations,
+      };
+    }
+
+    [[nodiscard]] auto warnings() const -> std::multiset<std::string> const& {
+      return warnings_;
+    }
+  };
+
+} // namespace routemon::datex2