From 17a3ea880402338420699e03bcb24181e4ff3924 Mon Sep 17 00:00:00 2001 From: Rutger Broekhoff Date: Thu, 2 May 2024 20:27:40 +0200 Subject: Initial commit Based on dc4ba6a --- lib/libtmi8/.envrc | 3 + lib/libtmi8/.gitignore | 3 + lib/libtmi8/Makefile | 41 + lib/libtmi8/flake.lock | 58 ++ lib/libtmi8/flake.nix | 42 + lib/libtmi8/include/tmi8/kv1_index.hpp | 135 +++ lib/libtmi8/include/tmi8/kv1_lexer.hpp | 46 + lib/libtmi8/include/tmi8/kv1_parser.hpp | 87 ++ lib/libtmi8/include/tmi8/kv1_types.hpp | 1528 ++++++++++++++++++++++++++++++ lib/libtmi8/include/tmi8/kv6_parquet.hpp | 46 + lib/libtmi8/src/kv1_index.cpp | 461 +++++++++ lib/libtmi8/src/kv1_lexer.cpp | 152 +++ lib/libtmi8/src/kv1_parser.cpp | 1258 ++++++++++++++++++++++++ lib/libtmi8/src/kv1_types.cpp | 773 +++++++++++++++ lib/libtmi8/src/kv6_parquet.cpp | 102 ++ 15 files changed, 4735 insertions(+) create mode 100644 lib/libtmi8/.envrc create mode 100644 lib/libtmi8/.gitignore create mode 100644 lib/libtmi8/Makefile create mode 100644 lib/libtmi8/flake.lock create mode 100644 lib/libtmi8/flake.nix create mode 100644 lib/libtmi8/include/tmi8/kv1_index.hpp create mode 100644 lib/libtmi8/include/tmi8/kv1_lexer.hpp create mode 100644 lib/libtmi8/include/tmi8/kv1_parser.hpp create mode 100644 lib/libtmi8/include/tmi8/kv1_types.hpp create mode 100644 lib/libtmi8/include/tmi8/kv6_parquet.hpp create mode 100644 lib/libtmi8/src/kv1_index.cpp create mode 100644 lib/libtmi8/src/kv1_lexer.cpp create mode 100644 lib/libtmi8/src/kv1_parser.cpp create mode 100644 lib/libtmi8/src/kv1_types.cpp create mode 100644 lib/libtmi8/src/kv6_parquet.cpp (limited to 'lib') diff --git a/lib/libtmi8/.envrc b/lib/libtmi8/.envrc new file mode 100644 index 0000000..4e0d702 --- /dev/null +++ b/lib/libtmi8/.envrc @@ -0,0 +1,3 @@ +use flake + +export DEVMODE=1 diff --git a/lib/libtmi8/.gitignore b/lib/libtmi8/.gitignore new file mode 100644 index 0000000..f6b8cf6 --- /dev/null +++ b/lib/libtmi8/.gitignore @@ -0,0 +1,3 @@ +src/*.o +libtmi8.a +libtmi8.so diff --git a/lib/libtmi8/Makefile b/lib/libtmi8/Makefile new file mode 100644 index 0000000..52a9807 --- /dev/null +++ b/lib/libtmi8/Makefile @@ -0,0 +1,41 @@ +# Taken from: +# Open Source Security Foundation (OpenSSF), “Compiler Options Hardening Guide +# for C and C++,” OpenSSF Best Practices Working Group. Accessed: Dec. 01, +# 2023. [Online]. Available: +# https://best.openssf.org/Compiler-Hardening-Guides/Compiler-Options-Hardening-Guide-for-C-and-C++.html +CXXFLAGS=-std=c++2b -g -fno-omit-frame-pointer -Iinclude $(if $(DEVMODE),-Werror,)\ + -O2 -Wall -Wformat=2 -Wconversion -Wtrampolines -Wimplicit-fallthrough \ + -U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=3 \ + -D_GLIBCXX_ASSERTIONS \ + -fstrict-flex-arrays=3 \ + -fstack-clash-protection -fstack-protector-strong +LDFLAGS=-larrow -lparquet -Wl,-z,defs \ + -Wl,-z,nodlopen -Wl,-z,noexecstack \ + -Wl,-z,relro -Wl,-z,now +DESTDIR=/usr/local + +LIBHDRS=include/tmi8/kv1_lexer.hpp include/tmi8/kv1_parser.hpp include/tmi8/kv1_types.hpp include/tmi8/kv6_parquet.hpp +LIBSRCS=src/kv1_index.cpp src/kv1_lexer.cpp src/kv1_parser.cpp src/kv1_types.cpp src/kv6_parquet.cpp +LIBOBJS=$(patsubst %.cpp,%.o,$(LIBSRCS)) + +.PHONY: all install libtmi8 clean +all: libtmi8 + +libtmi8: libtmi8.a libtmi8.so + +clean: + rm libtmi8.a libtmi8.so $(LIBOBJS) + +install: libtmi8.a $(LIBHDRS) + install -D -m644 include/tmi8/* -t $(DESTDIR)/include/tmi8 + install -D -m644 libtmi8.a -t $(DESTDIR)/lib + install -D -m644 libtmi8.so -t $(DESTDIR)/lib + +src/%.o: src/%.cpp $(LIBHDRS) + $(CXX) -c -o $@ $< $(CXXFLAGS) + +libtmi8.a: $(LIBOBJS) + $(AR) rcs $@ $^ + +libtmi8.so: $(LIBOBJS) + $(CXX) -shared -fPIC -o $@ $^ $(CXXFLAGS) $(LDFLAGS) diff --git a/lib/libtmi8/flake.lock b/lib/libtmi8/flake.lock new file mode 100644 index 0000000..5ff7d5d --- /dev/null +++ b/lib/libtmi8/flake.lock @@ -0,0 +1,58 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1701680307, + "narHash": "sha256-kAuep2h5ajznlPMD9rnQyffWG8EM/C73lejGofXvdM8=", + "rev": "4022d587cbbfd70fe950c1e2083a02621806a725", + "revCount": 88, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/numtide/flake-utils/0.1.88+rev-4022d587cbbfd70fe950c1e2083a02621806a725/018c340d-3287-7c66-818b-f2f646a808e3/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/numtide/flake-utils/0.1.88.tar.gz" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1701539137, + "narHash": "sha256-nVO/5QYpf1GwjvtpXhyxx5M3U/WN0MwBro4Lsk+9mL0=", + "rev": "933d7dc155096e7575d207be6fb7792bc9f34f6d", + "revCount": 552571, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/NixOs/nixpkgs/0.2311.552571+rev-933d7dc155096e7575d207be6fb7792bc9f34f6d/018c3242-a93c-7779-8d13-ddba0a38d24a/source.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://flakehub.com/f/NixOs/nixpkgs/*.tar.gz" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/lib/libtmi8/flake.nix b/lib/libtmi8/flake.nix new file mode 100644 index 0000000..2ae7fc9 --- /dev/null +++ b/lib/libtmi8/flake.nix @@ -0,0 +1,42 @@ +{ + inputs = { + nixpkgs.url = "https://flakehub.com/f/NixOs/nixpkgs/*.tar.gz"; + flake-utils.url = "https://flakehub.com/f/numtide/flake-utils/0.1.88.tar.gz"; + }; + + outputs = { self, nixpkgs, flake-utils, ... }@inputs: + flake-utils.lib.eachDefaultSystem + (system: + let + pkgs = import nixpkgs { + inherit system; + overlays = [ ]; + }; + + inherit (pkgs.gcc13) stdenv; + + oeuf-libtmi8 = stdenv.mkDerivation { + name = "oeuf-libtmi8"; + src = pkgs.lib.cleanSource ./.; + + nativeBuildInputs = with pkgs; [ gcc13 ]; + buildInputs = with pkgs; [ arrow-cpp boost182 ]; + buildPhase = '' + make libtmi8 + ''; + + installPhase = '' + make install DESTDIR="$out" + ''; + }; + in + { + packages.oeuf-libtmi8 = oeuf-libtmi8; + + devShells.default = pkgs.mkShell { + inputsFrom = [ oeuf-libtmi8 ]; + }; + + formatter = pkgs.nixpkgs-fmt; + }); +} diff --git a/lib/libtmi8/include/tmi8/kv1_index.hpp b/lib/libtmi8/include/tmi8/kv1_index.hpp new file mode 100644 index 0000000..621acf6 --- /dev/null +++ b/lib/libtmi8/include/tmi8/kv1_index.hpp @@ -0,0 +1,135 @@ +// vim:set sw=2 ts=2 sts et: + +#ifndef OEUF_LIBTMI8_KV1_INDEX_HPP +#define OEUF_LIBTMI8_KV1_INDEX_HPP + +#include + +#include + +#include + +struct Kv1Index { + Kv1Records *records; + + explicit Kv1Index(Kv1Records *records); + + std::unordered_map< + Kv1OrganizationalUnit::Key, + Kv1OrganizationalUnit *, + boost::hash> organizational_units; + std::unordered_map< + Kv1HigherOrganizationalUnit::Key, + Kv1HigherOrganizationalUnit *, + boost::hash> higher_organizational_units; + std::unordered_map< + Kv1UserStopPoint::Key, + Kv1UserStopPoint *, + boost::hash> user_stop_points; + std::unordered_map< + Kv1UserStopArea::Key, + Kv1UserStopArea *, + boost::hash> user_stop_areas; + std::unordered_map< + Kv1TimingLink::Key, + Kv1TimingLink *, + boost::hash> timing_links; + std::unordered_map< + Kv1Link::Key, + Kv1Link *, + boost::hash> links; + std::unordered_map< + Kv1Line::Key, + Kv1Line *, + boost::hash> lines; + std::unordered_map< + Kv1Destination::Key, + Kv1Destination *, + boost::hash> destinations; + std::unordered_map< + Kv1JourneyPattern::Key, + Kv1JourneyPattern *, + boost::hash> journey_patterns; + std::unordered_map< + Kv1ConcessionFinancerRelation::Key, + Kv1ConcessionFinancerRelation *, + boost::hash> concession_financer_relations; + std::unordered_map< + Kv1ConcessionArea::Key, + Kv1ConcessionArea *, + boost::hash> concession_areas; + std::unordered_map< + Kv1Financer::Key, + Kv1Financer *, + boost::hash> financers; + std::unordered_map< + Kv1JourneyPatternTimingLink::Key, + Kv1JourneyPatternTimingLink *, + boost::hash> journey_pattern_timing_links; + std::unordered_map< + Kv1Point::Key, + Kv1Point *, + boost::hash> points; + std::unordered_map< + Kv1PointOnLink::Key, + Kv1PointOnLink *, + boost::hash> point_on_links; + std::unordered_map< + Kv1Icon::Key, + Kv1Icon *, + boost::hash> icons; + std::unordered_map< + Kv1Notice::Key, + Kv1Notice *, + boost::hash> notices; + std::unordered_map< + Kv1TimeDemandGroup::Key, + Kv1TimeDemandGroup *, + boost::hash> time_demand_groups; + std::unordered_map< + Kv1TimeDemandGroupRunTime::Key, + Kv1TimeDemandGroupRunTime *, + boost::hash> time_demand_group_run_times; + std::unordered_map< + Kv1PeriodGroup::Key, + Kv1PeriodGroup *, + boost::hash> period_groups; + std::unordered_map< + Kv1SpecificDay::Key, + Kv1SpecificDay *, + boost::hash> specific_days; + std::unordered_map< + Kv1TimetableVersion::Key, + Kv1TimetableVersion *, + boost::hash> timetable_versions; + std::unordered_map< + Kv1PublicJourney::Key, + Kv1PublicJourney *, + boost::hash> public_journeys; + std::unordered_map< + Kv1PeriodGroupValidity::Key, + Kv1PeriodGroupValidity *, + boost::hash> period_group_validities; + std::unordered_map< + Kv1ExceptionalOperatingDay::Key, + Kv1ExceptionalOperatingDay *, + boost::hash> exceptional_operating_days; + std::unordered_map< + Kv1ScheduleVersion::Key, + Kv1ScheduleVersion *, + boost::hash> schedule_versions; + std::unordered_map< + Kv1PublicJourneyPassingTimes::Key, + Kv1PublicJourneyPassingTimes *, + boost::hash> public_journey_passing_times; + std::unordered_map< + Kv1OperatingDay::Key, + Kv1OperatingDay *, + boost::hash> operating_days; + + size_t size() const; +}; + +void kv1LinkRecords(Kv1Index &index); + +#endif // OEUF_LIBTMI8_KV1_INDEX_HPP diff --git a/lib/libtmi8/include/tmi8/kv1_lexer.hpp b/lib/libtmi8/include/tmi8/kv1_lexer.hpp new file mode 100644 index 0000000..df6a57c --- /dev/null +++ b/lib/libtmi8/include/tmi8/kv1_lexer.hpp @@ -0,0 +1,46 @@ +// vim:set sw=2 ts=2 sts et: + +#ifndef OEUF_LIBTMI8_KV1_LEXER_HPP +#define OEUF_LIBTMI8_KV1_LEXER_HPP + +#include +#include +#include +#include +#include +#include + +enum Kv1TokenType { + KV1_TOKEN_CELL, + KV1_TOKEN_ROW_END, +}; +struct Kv1Token { Kv1TokenType type; std::string data; }; + +struct Kv1Lexer { + std::vector errors; + std::vector tokens; + + explicit Kv1Lexer(std::string_view input); + + void lex(); + + private: + // Does not eat newline character. + void eatRestOfLine(); + void lexOptionalHeader(); + void lexOptionalComment(); + + static bool isWhitespace(int c); + + void readQuotedColumn(); + void readUnquotedColumn(); + void lexRow(); + // Returns true when a line ending was consumed. + bool eatWhitespace(); + + std::string_view input; + std::string_view slice; + std::string colbuf; +}; + +#endif // OEUF_LIBTMI8_KV1_LEXER_HPP diff --git a/lib/libtmi8/include/tmi8/kv1_parser.hpp b/lib/libtmi8/include/tmi8/kv1_parser.hpp new file mode 100644 index 0000000..ccd8ec6 --- /dev/null +++ b/lib/libtmi8/include/tmi8/kv1_parser.hpp @@ -0,0 +1,87 @@ +// vim:set sw=2 ts=2 sts et: + +#ifndef OEUF_LIBTMI8_KV1_PARSER_HPP +#define OEUF_LIBTMI8_KV1_PARSER_HPP + +#include +#include +#include +#include +#include + +#include +#include + +struct Kv1Parser { + explicit Kv1Parser(std::vector tokens, Kv1Records &parse_into); + + void parse(); + + private: + // Method pointer to a method of Kv1Parser (i.e. a function that takes + // 'this'; is not static) that takes no arguments and also does not return + // anything. + using ParseFunc = void (Kv1Parser::*)(); + static const std::unordered_map type_parsers; + + bool atEnd() const; + void eatRowEnds(); + const Kv1Token *cur() const; + const std::string *eatCell(std::string_view parsing_what); + std::string parseHeader(); + void eatRestOfRow(); + + void requireString(std::string_view field, bool mandatory, size_t max_length, std::string_view value); + std::optional requireBoolean(std::string_view field, bool mandatory, std::string_view value); + std::optional requireNumber(std::string_view field, bool mandatory, size_t max_digits, std::string_view value); + std::optional requireRgbColor(std::string_view field, bool mandatory, std::string_view value); + std::optional requireRdCoord(std::string_view field, bool mandatory, size_t min_digits, std::string_view value); + + std::string eatString(std::string_view field, bool mandatory, size_t max_length); + std::optional eatBoolean(std::string_view field, bool mandatory); + std::optional eatNumber(std::string_view field, bool mandatory, size_t max_digits); + std::optional eatRgbColor(std::string_view field, bool mandatory); + std::optional eatRdCoord(std::string_view field, bool mandatory, size_t min_digits); + + void parseOrganizationalUnit(); + void parseHigherOrganizationalUnit(); + void parseUserStopPoint(); + void parseUserStopArea(); + void parseTimingLink(); + void parseLink(); + void parseLine(); + void parseDestination(); + void parseJourneyPattern(); + void parseConcessionFinancerRelation(); + void parseConcessionArea(); + void parseFinancer(); + void parseJourneyPatternTimingLink(); + void parsePoint(); + void parsePointOnLink(); + void parseIcon(); + void parseNotice(); + void parseNoticeAssignment(); + void parseTimeDemandGroup(); + void parseTimeDemandGroupRunTime(); + void parsePeriodGroup(); + void parseSpecificDay(); + void parseTimetableVersion(); + void parsePublicJourney(); + void parsePeriodGroupValidity(); + void parseExceptionalOperatingDay(); + void parseScheduleVersion(); + void parsePublicJourneyPassingTimes(); + void parseOperatingDay(); + + size_t pos = 0; + std::vector tokens; + const std::chrono::time_zone *amsterdam = std::chrono::locate_zone("Europe/Amsterdam"); + + public: + std::vector warns; + std::vector global_errors; + std::vector record_errors; + Kv1Records &records; +}; + +#endif // OEUF_LIBTMI8_KV1_PARSER_HPP diff --git a/lib/libtmi8/include/tmi8/kv1_types.hpp b/lib/libtmi8/include/tmi8/kv1_types.hpp new file mode 100644 index 0000000..d4a0760 --- /dev/null +++ b/lib/libtmi8/include/tmi8/kv1_types.hpp @@ -0,0 +1,1528 @@ +// vim:set sw=2 ts=2 sts et: + +#ifndef OEUF_LIBTMI8_KV1_TYPES_HPP +#define OEUF_LIBTMI8_KV1_TYPES_HPP + +#include +#include +#include +#include +#include + +struct Kv1OrganizationalUnit; +struct Kv1HigherOrganizationalUnit; +struct Kv1UserStopPoint; +struct Kv1UserStopArea; +struct Kv1TimingLink; +struct Kv1Link; +struct Kv1Line; +struct Kv1Destination; +struct Kv1JourneyPattern; +struct Kv1ConcessionFinancerRelation; +struct Kv1ConcessionArea; +struct Kv1Financer; +struct Kv1JourneyPatternTimingLink; +struct Kv1Point; +struct Kv1PointOnLink; +struct Kv1Icon; +struct Kv1Notice; +struct Kv1NoticeAssignment; +struct Kv1TimeDemandGroup; +struct Kv1TimeDemandGroupRunTime; +struct Kv1PeriodGroup; +struct Kv1SpecificDay; +struct Kv1TimetableVersion; +struct Kv1PublicJourney; +struct Kv1PeriodGroupValidity; +struct Kv1ExceptionalOperatingDay; +struct Kv1ScheduleVersion; +struct Kv1PublicJourneyPassingTimes; +struct Kv1OperatingDay; + +struct Kv1Records { + std::vector organizational_units; + std::vector higher_organizational_units; + std::vector user_stop_points; + std::vector user_stop_areas; + std::vector timing_links; + std::vector links; + std::vector lines; + std::vector destinations; + std::vector journey_patterns; + std::vector concession_financer_relations; + std::vector concession_areas; + std::vector financers; + std::vector journey_pattern_timing_links; + std::vector points; + std::vector point_on_links; + std::vector icons; + std::vector notices; + std::vector notice_assignments; + std::vector time_demand_groups; + std::vector time_demand_group_run_times; + std::vector period_groups; + std::vector specific_days; + std::vector timetable_versions; + std::vector public_journeys; + std::vector period_group_validities; + std::vector exceptional_operating_days; + std::vector schedule_versions; + std::vector public_journey_passing_times; + std::vector operating_days; + + size_t size() const; +}; + +// These definitions implement TMI8, KV1 Dienstregeling (Timetable) version +// 8.3.0.2 (release), published by BISON on January 8, 2020. +// (Filename: tmi8 dienstregeling (kv 1) v8.3.0.2, release.docx) +// +// This specification and other BISON specifications, as well as other +// supplementary information, can be found on BISON's website: +// https://bison.dova.nu/ +// +// The specification that was used to create these definitions was downloaded +// from the following address: +// https://bison.dova.nu/sites/default/files/bestanden/tmi8_dienstregeling_kv_1_v8.3.0.2_release.pdf +// +// The KV1 table structure and the corresponding documentation describing the +// relevant tables and fields, as presented here, is derived from the original +// specification. Most documentation is a manually translated version of the +// documentation as present in the specification. The specification is licensed +// under CC BY-ND 3.0. The exact text of this license can be found on +// https://creativecommons.org/licenses/by-nd/3.0/nl/. + +// KV1 Table 1: Organizational Unit [ORUN] (MANDATORY) +// +// A collection of trips with the same validity features. An organizational +// unit can be part of a 'higher' unit. +// +// An organizational unit is defined as a unity vor which the planning of trips +// is compiled. When defining the organizational units, it is important that +// all trips within the package have a homogeneous validity (school holidays, +// shopping Sundays, foreign bank holidays). +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1OrganizationalUnit { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string organizational_unit_code; + + explicit Key(std::string data_owner_code, + std::string organizational_unit_code); + }; + + Key key; + // Mandatory, at most 50 characters. + std::string name; + // Mandatory, at most 10 characters. + std::string organizational_unit_type; + // Optional, at most 255 characters. + std::string description; +}; + +// KV1 Table 2: Higher Organizational Unit [ORUNORUN] (OPTIONAL) +// +// An in the hierarchy higher-ordered organizational unit for the purpose of +// (among others) recording of (deviating) validities on the high level. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1HigherOrganizationalUnit { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Parent, higher organizational unit + // that is referred to. + std::string organizational_unit_code_parent; + // Mandatory (key), at most 10 characters. Child, lower organizational unit. + std::string organizational_unit_code_child; + // Mandatory (key), at most 10 characters. [YYYY-MM-DD] Starting date of the + // hierarchical relation (can be a fixed value, e.g. 2006-12-31). + std::chrono::year_month_day valid_from; + + explicit Key(std::string data_owner_code, + std::string organizational_unit_code_parent, + std::string organizational_unit_code_child, + std::chrono::year_month_day valid_from); + }; + + Key key; + + Kv1OrganizationalUnit *p_organizational_unit_parent = nullptr; + Kv1OrganizationalUnit *p_organizational_unit_child = nullptr; +}; + +// KV1 Table 3: User Stop Point [USRSTOP] +// +// Stop or other point (e.g. Bridge, functioning as info for the bridge keeper) +// for which times are recorded in the planning system of the transit operator. +// +// Coordinates of a UserStopPoint are recorded as Point. When defining +// UserStopPoints, it is important that the coordinates can be unambiguously +// and verifiably recorded. For a stop, the coordinates of the stop sign are +// recorded. If there is no stop sign, the end of the bus stop (where the bus +// normally halts) is recorded as the coordinate of the stop. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1UserStopPoint { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Stop number in domain of operator. + std::string user_stop_code; + + explicit Key(std::string data_owner_code, + std::string user_stop_code); + }; + + Key key; + // Optional, at most 10 characters. Stop number in domain of integrator, + // (initially) equal to UserStopCode. + std::string timing_point_code; + // Mandatory, at most 5 characters. Boolean indicator whether USRSTOP is used + // as boarding stop, true by default. False for e.g. dummy stop for bridge + // keeper. + bool get_in = true; + // Mandatory, at most 5 characters. Boolean indicator whether USRSTOP is used + // as alighting stop. + bool get_out = false; + // Mandatory, at most 50 characters. Stop name. + std::string name; + // Mandatory, at most 50 characters. Town name. + std::string town; + // Optional, at most 10 characters. Reference to StopArea of which the + // UserStop is part. + std::string user_stop_area_code; + // Mandatory, at most 10 characters. Platform indication/letter. The '-' + // value is used to indication that this is not applicable. + std::string stop_side_code; + // Mandatory, at most 5 digits. Minimal stop duration for boarding and + // alighting, zero by default. In seconds. + double minimal_stop_time_s = 0; + // Optional, at most 3 digits. Length of stop platform. + std::optional stop_side_length; + // Optional, at most 255 characters. + std::string description; + // Mandatory, at most 10 characters. USRSTOPTYPE. Indicates the stop kind. + std::string user_stop_type; + // Optional, at most 30 characters. Nationally unique stop number. + std::string quay_code; + + Kv1UserStopArea *p_user_stop_area = nullptr; + Kv1Point *p_point = nullptr; +}; + +// KV1 Table 4: User Stop Area [USRSTAR] +// +// A StopArea is a collection of stops, which have the same name for passengers +// and logically belong together. (E.g. a bus station of transfer point.) Stops +// lying opposite each other can also form a StopArea. +// +// Used for display of all stops in a stop area on an overview display and for +// announcement of stop names (stops on both sides of the street share the same +// name). +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1UserStopArea { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Code of StopArea following coding + // of operator, e.g. PlaceCode. + std::string user_stop_area_code; + + explicit Key(std::string data_owner_code, + std::string user_stop_area_code); + }; + + Key key; + // Mandatory, at most 50 characters. + std::string name; + // Mandatory, at most 50 characters. + std::string town; + // Mandatory, at most 255 characters. + std::string description; +}; + +// KV1 Table 5: Timing Link [TILI] +// +// Link between two points which have the feature 'stop' or 'timing point'. A +// Timing Link is set between all stops and other timing points (e.g. for the +// bridge) which make part of a journey pattern. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1TimingLink { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Stop number in the domain of + // DataOwner (here: the operator). + std::string user_stop_code_begin; + // Mandatory (key), at most 10 characters. Stop number in the domain of + // DataOwner (here: the operator). + std::string user_stop_code_end; + + explicit Key(std::string data_owner_code, + std::string user_stop_code_begin, + std::string user_stop_code_end); + }; + + Key key; + // Optional, at most 5 digits. Minimal trip time (in seconds). + std::optional minimal_drive_time_s; + // Optional, at most 255 characters. + std::string description; + + Kv1UserStopPoint *p_user_stop_begin = nullptr; + Kv1UserStopPoint *p_user_stop_end = nullptr; +}; + +// KV1 Table 6: Link [LINK] +// +// A route link describes the connection between to points on the physical path +// of a route. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1Link { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Stop code in the domain of + // DataOwner (here: the operator). + std::string user_stop_code_begin; + // Mandatory (key), at most 10 characters. Stop code in the domain of + // DataOwner (here: the operator). + std::string user_stop_code_end; + // Mandatory (key), at most 5 characters. Modality for which the distance + // applies, see BISON enumeration E9. + // TODO: Check if BISON enumeration E9 can be put into an enum. + std::string transport_type; + + explicit Key(std::string data_owner_code, + std::string user_stop_code_begin, + std::string user_stop_code_end, + std::string transport_type); + }; + + Key key; + // Mandatory, at most 6 digits. Length of the link (in meters). + double distance = 0; + // Optional, at most 255 characters. + std::string description; + + Kv1UserStopPoint *p_user_stop_begin = nullptr; + Kv1UserStopPoint *p_user_stop_end = nullptr; +}; + +struct RgbColor { + uint8_t r, g, b = 0; +}; + +// KV1 Table 7: Line [LINE] +// +// A line is a collection of routes/journey patterns which is publically known +// under a shared number. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1Line { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Unique system line number in the + // domain of DataOwner. + std::string line_planning_number; + + explicit Key(std::string data_owner_code, + std::string line_planning_number); + }; + + Key key; + // Mandatory, at most 4 characters. Line number for the public, incl. S/N + // indications. + std::string line_public_number; + // Mandatory, at most 50 characters. + std::string line_name; + // Mandatory, at most three digits. Should be in the range [0, 400). + // Only processing Connexxion's KV1 export, however, shows us that this range + // constrained is not honored in practice. That is why we also don't care. + short line_ve_tag_number = 0; + // Optional, at most 255 characters. + std::string description; + // Mandatory, at most 5 characters. Modality, see BISON enumeration E9. + // TODO: Check if BISON enumeration E9 can be put into an enum. + std::string transport_type; + // Optional, at most 4 digits. Symbol / image for the line. Reference to ICON + // table. + std::optional line_icon; + // Optional, at most four characters. Background color for the line. + // Hexadecimal representation following RGB coding. Always six characters + // (RRGGBB), only numbers and/or capital letters. + std::optional line_color; + // Optional, at most four characters. Foreground color for the line. + // Hexadecimal representation following RGB coding. Always six characters + // (RRGGBB), only numbers and/or capital letters. + std::optional line_text_color; + + Kv1Icon *p_line_icon = nullptr; +}; + +// KV1 Table 8: Destination [DEST] +// +// A destination shows the place/district/description of the route for the +// passenger. Intermediate and detail destinations of a journey pattern are +// shown under a single desination code, together with the primary destination. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1Destination { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string dest_code; + + explicit Key(std::string data_owner_code, + std::string dest_code); + }; + + Key key; + // Mandatory, at most 50 characters. Full destination (e.g. compiled from + // primary, detail or intermediate destination). + std::string dest_name_full; + // Mandatory, at most 24 characters. Primary / intermediate destination in + // enumeration / final destination if 1 line is used. + std::string dest_name_main; + // Optional, at most 24 characters. Detail/secondary or intermediate + // destination for primary desination, final destination (for intermediate + // destination on line 1). + std::string dest_name_detail; + // Mandatory, at most 5 characters. Boolean which indcates whether + // DestNameDetail must always be shown (e.g. because this contains an + // important intermediate destination.) + bool relevant_dest_name_detail = false; + // Mandatory, at most 21 characters. Primary destination in 21 characters. + std::string dest_name_main_21; + // Optional, at most 21 characters. Detail/secondary/intermediate destination + // in 21 characters. + std::string dest_name_detail_21; + // Mandatory, at most 19 characters. Primary destination in 19 characters. + std::string dest_name_main_19; + // Optional, at most 19 characters. Detail/secondary/intermediate destination + // in 19 characters. + std::string dest_name_detail_19; + // Mandatory, at most 16 characters. Primary destination in 16 characters. + std::string dest_name_main_16; + // Optional, at most 16 characters. Detail/secondary/intermediate destination + // in 16 characters. + std::string dest_name_detail_16; + // Optional, at most 4 digits. Symbol/image for the destination. Reference to + // the ICON table. + std::optional dest_icon; + // Optional, at most 6 characters. Background color for the destination. + // Hexadecimal representation following RGB coding. Always six characters + // (RRGGBB), only six digits and/or capital letters. + std::optional dest_color; + // Optional, at most 30 characters (WTF?). Foreground color for the + // destination. Hexadecimal representation following RGB coding. Always six + // characters (RRGGBB), only six digits and/or capital letters. + std::optional dest_text_color; +}; + +// KV1 Table 9: Journey Pattern [JOPA] +// +// The journey pattern describes the route from start to end point as a ordered +// list of stops and links between stops/timing points. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1JourneyPattern { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string line_planning_number; + // Mandatory (key), at most 10 characters. + std::string journey_pattern_code; + + explicit Key(std::string data_owner_code, + std::string line_planning_number, + std::string journey_pattern_code); + }; + + Key key; + // Mandatory, at most 10 characters. Refers to a journey pattern type + // (JOPATYPE). + std::string journey_pattern_type; + // Mandatory, at most 1 character. One of [1, 2, A, B]. + char direction = 0; + // Optional, at most 255 characters. + std::string description; + + Kv1Line *p_line = nullptr; +}; + +// KV1 Table 10: Concession Financer Relation [CONFINREL] +// +// Concession financer relation (mainly parcel). Smallest unit for which data +// about a concession can be captured in relation to a financer and/or +// concession. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1ConcessionFinancerRelation { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Parcel code. + std::string con_fin_rel_code; + + explicit Key(std::string data_owner_code, + std::string con_fin_rel_code); + }; + + Key key; + // Mandatory, at most 10 characters. Concession code. + std::string concession_area_code; + // Optional, at most 10 characters. Code of financer/client of the parcel. + std::string financer_code; + + Kv1ConcessionArea *p_concession_area = nullptr; + Kv1Financer *p_financer = nullptr; +}; + +// KV1 Table 11: Concession Area [CONAREA] +// +// Concession (area). +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1ConcessionArea { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Code of the concession. + std::string concession_area_code; + + explicit Key(std::string data_owner_code, + std::string concession_area_code); + }; + + Key key; + // Mandatory, at most 255 characters. + std::string description; +}; + +// KV1 Table 12: Financer [FINANCER] (OPTIONAL) +// +// Financer of a parcel. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1Financer { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string financer_code; + + explicit Key(std::string data_owner_code, + std::string financer_code); + }; + + Key key; + // Mandatory, at most 255 characters. + std::string description; +}; + +// KV1 Table 13: Journey Pattern Timing Link [JOPATILI] +// +// Compilation of journey pattern from logical links (between pairs of +// stops/timing points). Features such as the destination code, the public line +// number, the concession financer relation (parcel) and product formula are +// set per connection. Moreover, a color and/or image linked to the line +// destination and the use of the (first) stop as boarding/alighting stop can +// be set per link. +// +// Timing Link: A timing link is a stop, set by the transit operator, where a +// bus / public transit vehicle may never depart earlier than set in the +// timetable. +// +// A logical link may never occur more than once in a journey pattern. +// Therefore, the combination of LinePlanningNumber, JourneyPatternCode, +// UserStopCodeBegin and UserStopCodeEnd must be unique in JOPATILI. +// +// The value of GetIn and GetOut are normally copied from the corresponding +// stop in the USRSTOP table, but can be overruled per journey pattern if so +// desired. +// +// A Icon or (Text)Color set here overrules the general value of the +// corresponding line (Line) or destination (Destination). +// +// A value of ShowFlexibleTrip or ProductFormulaType in PUJO or PUJOPASS +// overrules the value in JOPATILI. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1JourneyPatternTimingLink { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string line_planning_number; + // Mandatory (key), at most 10 characters. + std::string journey_pattern_code; + // Mandatory (key), at most 3 digits. + short timing_link_order = 0; + + explicit Key(std::string data_owner_code, + std::string line_planning_number, + std::string journey_pattern_code, + short timing_link_order); + }; + + Key key; + // Mandatory, at most 10 characters. Stop number in the domain of the + // DataOwner (here: the transit operator). + std::string user_stop_code_begin; + // Mandatory, at most 10 characters. Stop number in the domain of the + // DataOwner (here: the transit operator). + std::string user_stop_code_end; + // Mandatory, at most 10 characters. Concession financer relation / parcel + // (smallest unit). + std::string con_fin_rel_code; + // Mandatory, at most 10 characters. The destination (incl. intermediat + // destinations) as these are shown at the first stop of the journey pattern + // link. + std::string dest_code; + // Mandatory, at most 5 characters. Boolean which indicates whether the first + // stop of the connection is a timing stop. Indicator is at least "true" at + // first stop of a line and at waiting stops. + bool is_timing_stop = false; + // Optional, at most 4 characters. Public line number which must be shown on + // displays from the first stop of the journey pattern link (e.g. Line number + // + S). This is important when a deviating public line number applies from a + // certain point on forward. Normally, the public line number of the + // corresponding line is shown. + std::string display_public_line; + // Optional, at most 4 digits. Enumeration E10 (see section 2.5). A public + // transit service which distinguishes itself by a set of unique features, + // that is offered to the passenger as distinct (a marketing aspect). + // TODO: Check if we can turn BISON enumeration E10 into an enum + std::optional product_formula_type; + // Mandatory, at most 5 characters. Boolean indicator whether UserStopBegin + // is used as a boarding stop in this journey pattern. Usually equal to the + // value of the corresponding USRSTOP. + bool get_in = false; + // Mandatory, at most 5 characters. Boolean indicator whether UserStopBegin + // is used as an alighting stop in this journey pattern. Usually equal to the + // value of the corresponding USRSTOP. + bool get_out = false; + // Optional, at most 8 characters. Indicates whether the transit operator + // wants a not explicitly planned trip (i.e. a trip that only operates after + // reservation such as a 'call bus' (belbus), 'line taxi' (lijntaxi) etc.) to + // be shown on displays. Values according enumeration E21: TRUE (always), + // FALSE (never), REALTIME (only when tracking trip). + // TODO: Check if we can turn BISON enumeration E21 into an enum + std::string show_flexible_trip; + // Optional, at most 4 digits. Symbol / image for display of the line + // destination at the journey stop passing. Reference to the ICON table. + std::optional line_dest_icon; + // Optional, at most 6 characters. Background color for display of the line + // destination at a journey stop passing. Hexadecimal representation + // following RGB coding. Always six characters (RRGGBB), only numbers and/or + // capital letters. + std::optional line_dest_color; + // Optional, at most 6 characters. Foreground color for display of the line + // destination at a journey stop passing. Hexadecimal representation + // following RGB coding. Always six characters (RRGGBB), only numbers and/or + // capital letters. + std::optional line_dest_text_color; + + Kv1Line *p_line = nullptr; + Kv1JourneyPattern *p_journey_pattern = nullptr; + Kv1UserStopPoint *p_user_stop_begin = nullptr; + Kv1UserStopPoint *p_user_stop_end = nullptr; + Kv1ConcessionFinancerRelation *p_con_fin_rel = nullptr; + Kv1Destination *p_dest = nullptr; + Kv1Icon *p_line_dest_icon = nullptr; +}; + +// KV1 Table 14: Point [POINT] +// +// A point is the smallest location which can be reffered to within the public +// transit network. Every stop (USRSTOP) is a point. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1Point { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string point_code; + + explicit Key(std::string data_owner_code, + std::string point_code); + }; + + Key key; + // Mandatory, at most 10 characters. Refers to the POINTTYPE table. + std::string point_type; + // Mandatory, at most 10 characters. Refers to the GEOSYSTYPE table. Only + // allowed to have the value "RD" (rijkdsdriehoekstelsel; the national Dutch + // coordinate system). + std::string coordinate_system_type; + // Mandatory, at most 15 characters. X position in the RD coordinate system, + // in meters (at least 6 digits). + double location_x_ew = 0; + // Mandatory, at most 15 characters. Y position in the RD coordinate system, + // in meters (at least 6 digits). + double location_y_ns = 0; + // Optional, at most 15 characters. + // NOTE: the standart (presumeably wrongly) indicates this field as having + // alphanumeric contents. + std::optional location_z; + // Optional, at most 255 characters. + std::string description; +}; + +// KV1 Table 15: Point on Link [POOL] +// +// A point that is used to geographically describe the trajectory between two +// stops. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1PointOnLink { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Stop number in the domain of the + // DataOwner (here: transit operator). + std::string user_stop_code_begin; + // Mandatory (key), at most 10 characters. Stop number in the domain of the + // DataOwner (here: transit operator). + std::string user_stop_code_end; + // Mandatory (key), at most 10 characters. Code from the road manager for KAR + // points. For curve points of the DataOwner (often the transit operator). + std::string point_data_owner_code; + // Mandatory (key), at most 10 charcters. + std::string point_code; + // Mandatory (key), at most 5 characters. Modality for which the distance + // applies, see BISON enumeration E9. + std::string transport_type; + + explicit Key(std::string data_owner_code, + std::string user_stop_code_begin, + std::string user_stop_code_end, + std::string point_data_owner_code, + std::string point_code, + std::string transport_type); + }; + + Key key; + // Mandatory, at most 5 digits. Distance in meters relative to the start of + // the link. + double distance_since_start_of_link = 0; + // Optional, at most 4 digits. Crossing speed for a public transit vehicle + // from the previous point (on a link) in m/s. + std::optional segment_speed_mps = 0; + // Optional, at most 4 digits. Comfort speed for a public transit vehicle on + // the curve point. + std::optional local_point_speed_mps = 0; + // Optional, at most 255 characters. + std::string description; + + Kv1UserStopPoint *p_user_stop_begin = nullptr; + Kv1UserStopPoint *p_user_stop_end = nullptr; + Kv1Point *p_point = nullptr; +}; + +// KV1 Table 16: Icon [ICON] +// +// Table with images which can be referred to from DEST.DestIcon, LINE.LineIcon +// and JOPATILI.LineDestIcon to load the correct image. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1Icon { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 4 digits. Reference from other tables for the + // requested image. + short icon_number = 0; + + explicit Key(std::string data_owner_code, + short icon_number); + }; + + Key key; + // Mandatory, at most 1024 characters. Absolute URI to a publically available + // location from which the image can be loaded. The extension of the file + // indicates the image type. + // Supported file types are: GIF (.gif), JPEG (.jpg, .jpeg), + // PNG (.png), SVG (.svg) + // Supported protocols are: HTTP, HTTPS, FTP + // Prefer to not use any capital letters. Examples: + // - http://bison.dova.nu/images/logo.png + // - https://bison.dova.nu/images/logo.png + // - ftp://ftp.dova.nu/images/logo.png + std::string icon_uri; +}; + +// KV1 Table 17: Notice [NOTICE] (OPTIONAL) +// +// A (reusable) text with supplementary information about exceptions / +// clarifications for a line, journey pattern etc. +// +// Usage is optional; when there are no clarifying texts, the NOTICE table does +// not need to be provided in a KV1 set. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1Notice { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 20 characters. Identification of Notice (remark, + // clarifying text). + std::string notice_code; + + explicit Key(std::string data_owner_code, + std::string notice_code); + }; + + Key key; + // Mandatory, at most 1024 characters. Content, text. Contains contact + // information such as telephone number, web address and reservation time for + // 'call buses' (belbussen) and other demand-based transit. + std::string notice_content; +}; + +// KV1 Table 18: Notice Assignment [NTCASSGNM] (OPTIONAL) +// +// Linking table in which Notice (remark, clarfiying text) is assigned to a +// line, journey pattern, stops within a journey pattern, journey etc. Notice +// Assignment contains all logical key elements of the corresponding objects to +// which a Notice can be assigned. +// +// Different attributes are required for the Notice Assignment, depending on +// the type object to which the Notice is assigned. In the following table +// structure, this is indicated as 'Only relevant for ...'. This means that +// fields for other object types in the Notice Assignment can be ignored. +// +// Moreover, it can also occur that not all key fields of the linked table are +// of interest (content-wise) for recording the Notice. +// +// Both matters are summarised in this overview: +// +// -------------------------------------------------------- +// AssignedObject PUJO PUJOPASS LINE JOPATILI +// -------------------------------------------------------- +// DataOwnerCode........... x ...... x ...... x ..... x ... +// TimetableVersionCode ... o ............................. +// OrganizationalUnitCode . o ...... o .................... +// ScheduleCode .................... o .................... +// ScheduleTypeCode ................ o .................... +// PeriodGroupCode ........ o ............................. +// SpecificDayCode ........ o ............................. +// DayType ................ o ............................. +// LinePlanningNumber ..... x ...... x ...... x ..... x ... +// JourneyNumber .......... x ...... x .................... +// StopOrder ....................... o .............. o ... +// JourneyPatternCode ............................... x ... +// TimingLinkOrder .................................. o ... +// UserStopCode .................... o .............. o ... +// -------------------------------------------------------- +// +// Legend: +// x - Mandatory. The Notice for this object type is always depndent on the +// value of the attribute. +// o - Optional. The Notice can be independent of the value of this +// attribute for this object type. +// - Attribute is no key field for this object type and can be +// ignored when processed. +// +// Usage of Notice Assignment is optional in KV1. If there are no clarifying +// texts, then the Notice Assignment table is not required to be present in the +// provided KV1 set. +// +// This table is part of the core data tables, which are common for all KV1 +// variants. +struct Kv1NoticeAssignment { + // Mandatory, at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory, at most 20 characters. Notice that is assigned. + std::string notice_code; + // Mandatory, at most 8 characters. Object type to which Notice is assigned. + std::string assigned_object; + // Optional, at most 10 characters. Only relevant for PUJO. + std::string timetable_version_code; + // Optional, at most 10 characters. Only relevant for PUJO and PUJOPASS. + std::string organizational_unit_code; + // Optional, at most 10 characters. Only relevant for PUJOPASS. + std::string schedule_code; + // Optional, at most 10 characters. Only relevant for PUJOPASS. + std::string schedule_type_code; + // Optional, at most 10 characters. Only relevant for PUJO. + std::string period_group_code; + // Optional, at most 10 characters. Only relevant for PUJO. + std::string specific_day_code; + // Optional, at most 10 characters. Only relevant for PUJO. + // [0|1][0|2][0|3][0|4][0|5][0|6][0|7] for Mon, Tue, Wed, Thu, Fri, Sat, Sun. + // E.g. 1234500 means Mon, Tue, Wed, Thu, Fri but not Sat, Sun. + std::string day_type; + // Mandatory, at most 10 characters. Mandatory for all object types. + std::string line_planning_number; + // Optional (for all object types except PUJO and PUJOPASS), at most 6 + // digits. Only relevant for PUJO and PUJOPASS. Must be in the range + // [0-1000000). + std::optional journey_number; + // Optional, at most 4 digits. Only relevant for PUJOPASS and JOPATILI. + std::optional stop_order; + // Optional (for all object types except JOPATILI), at most 4 digits. Only + // relevant for JOPATILI. + std::string journey_pattern_code; + // Optional (at most 3 digits). Only relevant for JOPATILI. + std::optional timing_link_order; + // Optional (at most 10 characters). Only relevant for PUJOPASS and JOPATILI. + // For JOPATILI, this correspond to the first stop of the link. + std::string user_stop_code; + + Kv1Notice *p_notice = nullptr; +}; + +// KV1 Table 19: Time Demand Group [TIMDEMGRP] +// +// A time demand group is a grouping of the run time distribution from stop to +// stop, for a journey pattern (from start to end point). +// +// This table is part of the KV1 variant "validities and time demand groups". +struct Kv1TimeDemandGroup { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string line_planning_number; + // Mandatory (key), at most 10 characters. Refers to the JOPATILI table. + std::string journey_pattern_code; + // Mandatory (key), at most 10 characters. Defines the code for the time + // demand group. (NOTE: this is not entirely made clear by the specification. + // This claim must be verified.) + std::string time_demand_group_code; + + explicit Key(std::string data_owner_code, + std::string line_planning_number, + std::string journey_pattern_code, + std::string time_demand_group_code); + }; + + Key key; + + Kv1Line *p_line = nullptr; + Kv1JourneyPattern *p_journey_pattern = nullptr; +}; + +// KV1 Table 20: Time Demand Group Run Time [TIMDEMRNT] +// +// The run time structure/distribution for all timing links of a journey +// pattern or a time demand group. +// +// Optional run time elements are, when these are present, used to more +// accurately calculate expected departure times based on punctuality +// deviations. +// +// This table is part of the KV1 variant "validities and time demand groups". +struct Kv1TimeDemandGroupRunTime { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string line_planning_number; + // Mandatory (key), at most 10 characters. Refers to the JOPATILI table. + std::string journey_pattern_code; + // Mandatory (key), at most 10 characters. Refers to the TIMDEMGRP table. + std::string time_demand_group_code; + // Mandatory (key), at most 3 digits. Reference number of a link within the + // journey pattern (a link can occur more than once within a journey + // pattern). + short timing_link_order = 0; + + explicit Key(std::string data_owner_code, + std::string line_planning_number, + std::string journey_pattern_code, + std::string time_demand_group_code, + short timing_link_order); + }; + + Key key; + // Mandatory, at most 10 characters. Refers to the first stop of the link. + std::string user_stop_code_begin; + // Mandatory, at most 10 characters. Refers to the last stop of the link. + std::string user_stop_code_end; + // Mandatory, at most 5 digits. Planned total run time on link for time + // demand group: (Departure time end stop - departure time begin stop) + // corresponding to the time demand group. In seconds. + double total_drive_time_s = 0; + // Mandatory, at most 5 digits. Planned minimal run time on link for time + // demand group. Often calculated as: (Arrival time end stop - arrival time + // begin stop) corresponding to the time demand group. In seconds. + double drive_time_s = 0; + // Optional, at most 5 digits. Expected/planned delay/congestion on link for + // time demand group. In seconds. + std::optional expected_delay_s; + // Optional, at most 5 digits. Layover/catch-up time. Gives play in the + // timetable. In seconds. + // LayOverTime = TotDriveTime - DriveTime + ExpectedDelay - StopWaitTime. + std::optional layover_time; + // Mandatory, at most 5 digits. Planned stop waiting time at the final stop + // of the link for the time demand group. Determined based on the difference + // between the departure time and arrival time at this stop. Is zero when no + // waiting time is planned for this stop. In seconds. + double stop_wait_time = 0; + // Optional, at most 5 digits. Planned minimal stop time for + // boarding/alighting of passengers at the final stop of the link for the + // time demand group. Application: at hub stops with a planned waiting time, + // the difference between the planned waiting time and the minimum stop time + // is the layover/catch-up time. In seconds. + std::optional minimum_stop_time; + + Kv1Line *p_line = nullptr; + Kv1UserStopPoint *p_user_stop_begin = nullptr; + Kv1UserStopPoint *p_user_stop_end = nullptr; + Kv1JourneyPattern *p_journey_pattern = nullptr; + Kv1TimeDemandGroup *p_time_demand_group = nullptr; + Kv1JourneyPatternTimingLink *p_journey_pattern_timing_link = nullptr; +}; + +// KV1 Table 21: Period Group [PEGR] +// +// Period group is an indication of a 'homogeneous period' during the year, +// i.e. a period in which the schedule has the same composition w.r.t. +// frequencies and run times. +// +// This table is part of the KV1 variant "validities and time demand groups". +struct Kv1PeriodGroup { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string period_group_code; + + explicit Key(std::string data_owner_code, + std::string period_group_code); + }; + + Key key; + // Optional, at most 255 characters. + std::string description; +}; + +// KV1 Table 22: Specific Day [SPECDAY] +// +// A specific day is a feature of a day for which a deviating service level is +// provided, respective to a normal day of the week. +// +// E.g. shopping Sundays (koopzondagen, if not every Sunday), New Year's Eve +// (oudejaarsdag), foreign bank holidays (as applicable). +// +// This table is part of the KV1 variant "validities and time demand groups". +struct Kv1SpecificDay { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Default: "NORMAL". + std::string specific_day_code; + + explicit Key(std::string data_owner_code, + std::string specific_day_code); + }; + + Key key; + // Mandatory, at most 50 characters. + std::string name; + // Optional, at most 255 characters. + std::string description; +}; + +// KV1 Table 23: Timetable Version [TIVE] +// +// A timetable version budles all planned activities for an organizational +// unit. For the public schedule, these are trips, routes, run times etc. +// +// When processing a new Timetable Version, it is checked if another TIVE with +// the same key has already been processed. If this is the case, ValidFrom must +// be equal to the starting date of the previously provided set. The new set +// replaces the older one. A package with a new starting date is only processed +// if another TimetableVersionCode is used. +// +// This table is part of the KV1 variant "validities and time demand groups". +struct Kv1TimetableVersion { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string organizational_unit_code; + // Mandatory (key), at most 10 characters. + std::string timetable_version_code; + // Mandatory (key), at most 10 charactes. + std::string period_group_code; + // Mandatory (key), at most 10 characters. Default: "NORMAL". + std::string specific_day_code; + + explicit Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string timetable_version_code, + std::string period_group_code, + std::string specific_day_code); + }; + + Key key; + // Mandatory, at most 10 characters. Datum on which the timetable goes into + // effect, following the YYYY-MM-DD format. + std::chrono::year_month_day valid_from; + // Mandatory, at most 10 characters. Value: "PUBT". + std::string timetable_version_type; + // Optional, at most 10 characters. Datum on which the timetable goes out of + // effect, following the YYYY-MM-DD format. + std::optional valid_thru; + // Optional, at most 255 characters. Should be null/empty. + std::string description; + + Kv1OrganizationalUnit *p_organizational_unit = nullptr; + Kv1PeriodGroup *p_period_group = nullptr; + Kv1SpecificDay *p_specific_day = nullptr; +}; + +// KV1 Table 24: Public Journey [PUJO] +// +// Public journeys are journeys that are operated by a public transit +// organization and are accessible to the passenger. +// +// Business rules: +// - If ShowFlexibleTrip or ProductFormulaType is set in a record of this +// table, this takes precedence over the value as in the corresponding +// JOPATILI entry. +// +// This table is part of the KV1 variant "validities and time demand groups". +struct Kv1PublicJourney { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string timetable_version_code; + // Mandatory (key), at most 10 characters. + std::string organizational_unit_code; + // Mandatory (key), at most 10 characters. + std::string period_group_code; + // Mandatory (key), at most 10 characters. + std::string specific_day_code; + // Mandatory (key), at most 7 characters. + // [0|1][0|2][0|3][0|4][0|5][0|6][0|7] for Mon, Tue, Wed, Thu, Fri, Sat, Sun. + // E.g. 1234500 means Mon, Tue, Wed, Thu, Fri but not Sat, Sun. + // TODO: See if we can make this into a more concrete type + std::string day_type; + // Mandatory (key), at most 10 characters. + std::string line_planning_number; + // Mandatory (key), at most 6 digits. Must be in the range [0-1000000). + int journey_number = 0; + + explicit Key(std::string data_owner_code, + std::string timetable_version_code, + std::string organizational_unit_code, + std::string period_group_code, + std::string specific_day_code, + std::string day_type, + std::string line_planning_number, + int journey_number); + }; + + Key key; + // Mandatory, at most 10 characters. + std::string time_demand_group_code; + // Mandatory, at most 10 characters. + std::string journey_pattern_code; + // Mandatory, at most 8 characters. Format: "HH:MM:SS". + std::chrono::hh_mm_ss departure_time; + // Mandatory, at most 13 characters. Values as in BISON enumeration E3. + // Allowed are: "ACCESSIBLE", "NOTACCESSIBLE" and "UNKNOWN". + // TODO: See if we can fit BISON enumeration E3 into an enum + std::string wheelchair_accessible; + // Mandatory, at most 5 characters. Boolean. Value "true": journey is + // operator by DataOwner. Value "false": journey is operator by a different + // DataOwner. Indicator is meant for a line that is operated jointly by + // multiple transit operators. The indicator is used to be able to match the + // journey operation (KV6, KV19 etc.); only journeys for which the indicator + // is "true" can be expected to have corresponding current/real-time + // information, although "true" doesn't necessarily mean that this + // current/real-time information will (always) become available. + bool data_owner_is_operator = false; + // Mandatory, at most 5 characters. Boolean. Indicates whether + // current/real-time journey information may be expected for the + // corresponding journey ("true" or "false"). + bool planned_monitored = false; + // Optional, at most 4 digits. BISON enumeration E10. Intended to allow + // capturing transit mode features at the journey level. + // TODO: See if we can make BISON enumeration E10 into an enum + std::optional product_formula_type; + // Optional, at most 8 characters. Indicates whether the transit operator + // wants that a not-explicitly planned trip (i.e. a journey that only runs on + // reservation, e.g. 'call bus' (belbus), 'line taxi' (lijntaxi) etc.) to be + // shown on displays. Values following BISON enumeration E21: TRUE (always), + // FALSE (never), REALTIME (only when journey is tracked). + // TODO: See if we can make BISON enumeration E21 into an enum + std::string show_flexible_trip; + + Kv1TimetableVersion *p_timetable_version = nullptr; + Kv1OrganizationalUnit *p_organizational_unit = nullptr; + Kv1PeriodGroup *p_period_group = nullptr; + Kv1SpecificDay *p_specific_day = nullptr; + Kv1Line *p_line = nullptr; + Kv1TimeDemandGroup *p_time_demand_group = nullptr; + Kv1JourneyPattern *p_journey_pattern = nullptr; +}; + +// KV1 Table 25: Period Group Validity [PEGRVAL] +// +// Validities (multiple from-thru data) of a period group. +// +// This table is part of the KV1 variant "validities and time demand groups". +struct Kv1PeriodGroupValidity { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string organizational_unit_code; + // Mandatory (key), at most 10 characters. + std::string period_group_code; + // Mandatory (key), at most 10 characters. Date of the start of the validity + // period. Format: "YYYY-MM-DD". + std::chrono::year_month_day valid_from; + + explicit Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string period_group_code, + std::chrono::year_month_day valid_from); + }; + + Key key; + // Mandatory, at most 10 characters. Date of the end of the validity period. + // Format: "YYYY-MM-DD". + std::chrono::year_month_day valid_thru; + + Kv1OrganizationalUnit *p_organizational_unit = nullptr; + Kv1PeriodGroup *p_period_group = nullptr; +}; + +// KV1 Table 26: Exceptional Operating Day [EXCOPDAY] +// +// Contains exceptional validity dates, for which the service runs following a +// different day type (such as another day of the week or a different period). +// +// This table is part of the KV1 variant "validities and time demand groups". +struct Kv1ExceptionalOperatingDay { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. Organization unit for which an + // exceptional day validity applies. + std::string organizational_unit_code; + // Mandatory (key), at most 23 characters. Date (+ time) for which the + // exceptional validity applies. Format: "YYYYMMDDThh:mm:ssTZD". + std::chrono::sys_seconds valid_date; + + explicit Key(std::string data_owner_code, + std::string organizational_unit_code, + std::chrono::sys_seconds valid_date); + }; + + Key key; + // Mandatory, at most 7 characters. The exceptional day type that applies on + // a calendar day: [0|1][0|2][0|3][0|4][0|5][0|6][0|7] for Mon, Tue, Wed, + // Thu, Fri, Sat. + // E.g. 1234500 means Mon, Tue, Wed, Thu, Fri but not Sat, Sun. + // TODO: See if we can make this into a more concrete type + std::string day_type_as_on; + // Mandatory, at most 10 characters. Specific day service level to which the + // exceptional day validity refers. + std::string specific_day_code; + // Optional, at most 10 characters. An exceptional day validity can be + // related to the service level of another period (e.g. the school holiday + // schedule). This exceptional period reference is set here. + // + // E.g. on Good Friday or the day after Ascension day, transit runs according + // to the holiday season schedule, while transit runs following the winter + // package in the surrounding days. + std::string period_group_code; + // Optional, at most 255 characters. + std::string description; + + Kv1OrganizationalUnit *p_organizational_unit = nullptr; + Kv1SpecificDay *p_specific_day = nullptr; + Kv1PeriodGroup *p_period_group = nullptr; +}; + +// KV1 Table 27: Schedule Version [SCHEDVERS] +// +// A schedule version bundles the planned activities for an organisation unit +// per day type. The journeys with passing times and corresponding routes are +// for the public timetable. +// +// When processing a new Schedule Version, it is checked if another SCHEDVERS +// with the same key has already been processed. If this is the case, ValidFrom +// must be equal to the starting date of the previously provided set. The new +// set replaces the older one. A package with a new starting date is only +// processed if another Schedule Code is used. +// +// This table is part of the KV1 variant "schedules and passing times". +struct Kv1ScheduleVersion { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string organizational_unit_code; + // Mandatory (key), at most 10 characters. A unique code in combination with + // the ScheduleTypeCode of the package within the ORUN. + std::string schedule_code; + // Mandatory (key), at most 10 characters. Code for the Schedule Type (Day Type). + std::string schedule_type_code; + + explicit Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string schedule_code, + std::string schedule_type_code); + }; + + Key key; + // Mandatory, at most 10 characters. Date on which the schedule goes into + // effect. Format: "YYYY-MM-DD". + std::chrono::year_month_day valid_from; + // Optional, at most 10 characters. Date on which the schedule goes out of + // effect. Format: "YYYY-MM-DD". + std::optional valid_thru; + // Optional, at most 255 characters. Should be empty/null. + std::string description; + + Kv1OrganizationalUnit *p_organizational_unit = nullptr; +}; + +// KV1 Table 28: Public Journey Passing Times [PUJOPASS] +// +// Public journey with arrival and departure times at all stops (and other +// timing points). +// +// Business rules: +// - If ShowFlexibleTrip or ProductFormulaType is set here, then this takes +// precedence over the value in the corresponding JOPATILI record. +// - All stop passings of a public journey refer to the same journey pattern +// (JOPA)! +// +// This table is part of the KV1 variant "schedules and passing times". +struct Kv1PublicJourneyPassingTimes { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string organizational_unit_code; + // Mandatory (key), at most 10 characters. A unique code in combination with + // the ScheduleTypeCode of the package within the ORUN. + std::string schedule_code; + // Mandatory (key), at most 10 characters. Code for the Schedule Type (e.g. + // Day Type). + std::string schedule_type_code; + // Mandatory (key), at most 10 characters. + std::string line_planning_number; + // Mandatory (key), at most 6 digits. Must be in the range [0-1000000). + int journey_number = 0; + // Mandatory (key), at most 4 digits. + short stop_order = 0; + + explicit Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string schedule_code, + std::string schedule_type_code, + std::string line_planning_number, + int journey_number, + short stop_order); + }; + + Key key; + // Mandatory, at most 10 characters. + std::string journey_pattern_code; + // Mandatory, at most 10 characters. + std::string user_stop_code; + // Mandatory (except for the first stop of a journey), at most 8 digits. Not + // compulsory for the first stop of a journey. Format: "HH:MM:SS". + std::optional> target_arrival_time; + // Mandatory (expect for the last stop of a journey), at most 8 digits. Not + // compulsory for the last stop of a journey. Format: "HH:MM:SS". + std::optional> target_departure_time; + // Mandatory, at most 13 characters. Values as in BISON enumeration E3. + // Allowed are: "ACCESSIBLE", "NOTACCESSIBLE" and "UNKNOWN". + // TODO: See if we can fit BISON enumeration E3 into an enum + std::string wheelchair_accessible; + // Mandatory, at most 5 characters. Boolean. Value "true": journey is + // operator by DataOwner. Value "false": journey is operator by a different + // DataOwner. Indicator is meant for a line that is operated jointly by + // multiple transit operators. The indicator is used to be able to match the + // journey operation (KV6, KV19 etc.); only journeys for which the indicator + // is "true" can be expected to have corresponding current/real-time + // information, although "true" doesn't necessarily mean that this + // current/real-time information will (always) become available. + bool data_owner_is_operator = false; + // Mandatory, at most 5 characters. Boolean. Indicates whether + // current/real-time journey information may be expected for the + // corresponding journey ("true" or "false"). + bool planned_monitored = false; + // Optional, at most 4 digits. BISON enumeration E10. Intended to allow + // capturing transit mode features at the journey level. + // TODO: See if we can make BISON enumeration E10 into an enum + std::optional product_formula_type; + // Optional, at most 8 characters. Indicates whether the transit operator + // wants that a not-explicitly planned trip (i.e. a journey that only runs on + // reservation, e.g. 'call bus' (belbus), 'line taxi' (lijntaxi) etc.) to be + // shown on displays. Values following BISON enumeration E21: TRUE (always), + // FALSE (never), REALTIME (only when journey is tracked). + // TODO: See if we can make BISON enumeration E21 into an enum + std::string show_flexible_trip; + + Kv1OrganizationalUnit *p_organizational_unit = nullptr; + Kv1ScheduleVersion *p_schedule_version = nullptr; + Kv1Line *p_line = nullptr; + Kv1JourneyPattern *p_journey_pattern = nullptr; + Kv1UserStopPoint *p_user_stop = nullptr; +}; + +// KV1 Table 29: Operating Day [OPERDAY] +// +// Contains the operational calendar. Which package (schedule version) applies +// is specified per day, per organisation unit. +// +// This table is part of the KV1 variant "schedules and passing times". +struct Kv1OperatingDay { + struct Key { + // Mandatory (key), at most 10 characters. Transport operator (from list as + // defined in BISON enumeration E1). + std::string data_owner_code; + // Mandatory (key), at most 10 characters. + std::string organizational_unit_code; + // Mandatory (key), at most 10 characters. + std::string schedule_code; + // Mandatory (key), at most 10 characters. + std::string schedule_type_code; + // Mandatory (key), at most 10 characters. Date on which the package + // (schedule version) applies. Format: "YYYY-MM-DD". + std::chrono::year_month_day valid_date; + + explicit Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string schedule_code, + std::string schedule_type_code, + std::chrono::year_month_day valid_date); + }; + + Key key; + // Optional, at most 255 characters. + std::string description; + + Kv1OrganizationalUnit *p_organizational_unit = nullptr; + Kv1ScheduleVersion *p_schedule_version = nullptr; +}; + +bool operator==(const Kv1OrganizationalUnit::Key &a, const Kv1OrganizationalUnit::Key &b); +bool operator==(const Kv1HigherOrganizationalUnit::Key &a, const Kv1HigherOrganizationalUnit::Key &b); +bool operator==(const Kv1UserStopPoint::Key &a, const Kv1UserStopPoint::Key &b); +bool operator==(const Kv1UserStopArea::Key &a, const Kv1UserStopArea::Key &b); +bool operator==(const Kv1TimingLink::Key &a, const Kv1TimingLink::Key &b); +bool operator==(const Kv1Link::Key &a, const Kv1Link::Key &b); +bool operator==(const Kv1Line::Key &a, const Kv1Line::Key &b); +bool operator==(const Kv1Destination::Key &a, const Kv1Destination::Key &b); +bool operator==(const Kv1JourneyPattern::Key &a, const Kv1JourneyPattern::Key &b); +bool operator==(const Kv1ConcessionFinancerRelation::Key &a, const Kv1ConcessionFinancerRelation::Key &b); +bool operator==(const Kv1ConcessionArea::Key &a, const Kv1ConcessionArea::Key &b); +bool operator==(const Kv1Financer::Key &a, const Kv1Financer::Key &b); +bool operator==(const Kv1JourneyPatternTimingLink::Key &a, const Kv1JourneyPatternTimingLink::Key &b); +bool operator==(const Kv1Point::Key &a, const Kv1Point::Key &b); +bool operator==(const Kv1PointOnLink::Key &a, const Kv1PointOnLink::Key &b); +bool operator==(const Kv1Icon::Key &a, const Kv1Icon::Key &b); +bool operator==(const Kv1Notice::Key &a, const Kv1Notice::Key &b); +bool operator==(const Kv1TimeDemandGroup::Key &a, const Kv1TimeDemandGroup::Key &b); +bool operator==(const Kv1TimeDemandGroupRunTime::Key &a, const Kv1TimeDemandGroupRunTime::Key &b); +bool operator==(const Kv1PeriodGroup::Key &a, const Kv1PeriodGroup::Key &b); +bool operator==(const Kv1SpecificDay::Key &a, const Kv1SpecificDay::Key &b); +bool operator==(const Kv1TimetableVersion::Key &a, const Kv1TimetableVersion::Key &b); +bool operator==(const Kv1PublicJourney::Key &a, const Kv1PublicJourney::Key &b); +bool operator==(const Kv1PeriodGroupValidity::Key &a, const Kv1PeriodGroupValidity::Key &b); +bool operator==(const Kv1ExceptionalOperatingDay::Key &a, const Kv1ExceptionalOperatingDay::Key &b); +bool operator==(const Kv1ScheduleVersion::Key &a, const Kv1ScheduleVersion::Key &b); +bool operator==(const Kv1PublicJourneyPassingTimes::Key &a, const Kv1PublicJourneyPassingTimes::Key &b); +bool operator==(const Kv1OperatingDay::Key &a, const Kv1OperatingDay::Key &b); + +size_t hash_value(const Kv1OrganizationalUnit::Key &k); +size_t hash_value(const Kv1HigherOrganizationalUnit::Key &k); +size_t hash_value(const Kv1UserStopPoint::Key &k); +size_t hash_value(const Kv1UserStopArea::Key &k); +size_t hash_value(const Kv1TimingLink::Key &k); +size_t hash_value(const Kv1Link::Key &k); +size_t hash_value(const Kv1Line::Key &k); +size_t hash_value(const Kv1Destination::Key &k); +size_t hash_value(const Kv1JourneyPattern::Key &k); +size_t hash_value(const Kv1ConcessionFinancerRelation::Key &k); +size_t hash_value(const Kv1ConcessionArea::Key &k); +size_t hash_value(const Kv1Financer::Key &k); +size_t hash_value(const Kv1JourneyPatternTimingLink::Key &k); +size_t hash_value(const Kv1Point::Key &k); +size_t hash_value(const Kv1PointOnLink::Key &k); +size_t hash_value(const Kv1Icon::Key &k); +size_t hash_value(const Kv1Notice::Key &k); +size_t hash_value(const Kv1TimeDemandGroup::Key &k); +size_t hash_value(const Kv1TimeDemandGroupRunTime::Key &k); +size_t hash_value(const Kv1PeriodGroup::Key &k); +size_t hash_value(const Kv1SpecificDay::Key &k); +size_t hash_value(const Kv1TimetableVersion::Key &k); +size_t hash_value(const Kv1PublicJourney::Key &k); +size_t hash_value(const Kv1PeriodGroupValidity::Key &k); +size_t hash_value(const Kv1ExceptionalOperatingDay::Key &k); +size_t hash_value(const Kv1ScheduleVersion::Key &k); +size_t hash_value(const Kv1PublicJourneyPassingTimes::Key &k); +size_t hash_value(const Kv1OperatingDay::Key &k); + +#endif // OEUF_LIBTMI8_KV1_TYPES_HPP diff --git a/lib/libtmi8/include/tmi8/kv6_parquet.hpp b/lib/libtmi8/include/tmi8/kv6_parquet.hpp new file mode 100644 index 0000000..33b57ca --- /dev/null +++ b/lib/libtmi8/include/tmi8/kv6_parquet.hpp @@ -0,0 +1,46 @@ +// vim:set sw=2 ts=2 sts et: + +#ifndef OEUF_LIBTMI8_KV6_PARQUET_HPP +#define OEUF_LIBTMI8_KV6_PARQUET_HPP + +#include + +#include +#include +#include + +static const size_t MAX_PARQUET_CHUNK = 10000; + +struct ParquetBuilder { + ParquetBuilder(); + arrow::Result> getTable(); + + std::shared_ptr schema; + + arrow::StringBuilder types; + arrow::StringBuilder data_owner_codes; + arrow::StringBuilder line_planning_numbers; + arrow::Date32Builder operating_days; + arrow::UInt32Builder journey_numbers; + arrow::UInt8Builder reinforcement_numbers; + arrow::TimestampBuilder timestamps{arrow::timestamp(arrow::TimeUnit::SECOND), arrow::default_memory_pool()}; + arrow::StringBuilder sources; + arrow::Int16Builder punctualities; + arrow::StringBuilder user_stop_codes; + arrow::UInt16Builder passage_sequence_numbers; + arrow::UInt32Builder vehicle_numbers; + arrow::UInt32Builder block_codes; + arrow::StringBuilder wheelchair_accessibles; + arrow::UInt8Builder number_of_coaches; + arrow::Int32Builder rd_ys; + arrow::Int32Builder rd_xs; + arrow::UInt32Builder distance_since_last_user_stops; +}; + +[[nodiscard]] +arrow::Status writeArrowRecordsAsParquetFile(arrow::RecordBatchReader &rbr, std::filesystem::path filename); + +[[nodiscard]] +arrow::Status writeArrowTableAsParquetFile(const arrow::Table &table, std::filesystem::path filename); + +#endif // OEUF_LIBTMI8_KV6_PARQUET_HPP diff --git a/lib/libtmi8/src/kv1_index.cpp b/lib/libtmi8/src/kv1_index.cpp new file mode 100644 index 0000000..23e9596 --- /dev/null +++ b/lib/libtmi8/src/kv1_index.cpp @@ -0,0 +1,461 @@ +// vim:set sw=2 ts=2 sts et: + +#include + +Kv1Index::Kv1Index(Kv1Records *records) : records(records) { + organizational_units.reserve(records->organizational_units.size()); + for (size_t i = 0; i < records->organizational_units.size(); i++) { + auto *it = &records->organizational_units[i]; + organizational_units[it->key] = it; + } + higher_organizational_units.reserve(records->higher_organizational_units.size()); + for (size_t i = 0; i < records->higher_organizational_units.size(); i++) { + auto *it = &records->higher_organizational_units[i]; + higher_organizational_units[it->key] = it; + } + user_stop_points.reserve(records->user_stop_points.size()); + for (size_t i = 0; i < records->user_stop_points.size(); i++) { + auto *it = &records->user_stop_points[i]; + user_stop_points[it->key] = it; + } + user_stop_areas.reserve(records->user_stop_areas.size()); + for (size_t i = 0; i < records->user_stop_areas.size(); i++) { + auto *it = &records->user_stop_areas[i]; + user_stop_areas[it->key] = it; + } + timing_links.reserve(records->timing_links.size()); + for (size_t i = 0; i < records->timing_links.size(); i++) { + auto *it = &records->timing_links[i]; + timing_links[it->key] = it; + } + links.reserve(records->links.size()); + for (size_t i = 0; i < records->links.size(); i++) { + auto *it = &records->links[i]; + links[it->key] = it; + } + lines.reserve(records->lines.size()); + for (size_t i = 0; i < records->lines.size(); i++) { + auto *it = &records->lines[i]; + lines[it->key] = it; + } + destinations.reserve(records->destinations.size()); + for (size_t i = 0; i < records->destinations.size(); i++) { + auto *it = &records->destinations[i]; + destinations[it->key] = it; + } + journey_patterns.reserve(records->journey_patterns.size()); + for (size_t i = 0; i < records->journey_patterns.size(); i++) { + auto *it = &records->journey_patterns[i]; + journey_patterns[it->key] = it; + } + concession_financer_relations.reserve(records->concession_financer_relations.size()); + for (size_t i = 0; i < records->concession_financer_relations.size(); i++) { + auto *it = &records->concession_financer_relations[i]; + concession_financer_relations[it->key] = it; + } + concession_areas.reserve(records->concession_areas.size()); + for (size_t i = 0; i < records->concession_areas.size(); i++) { + auto *it = &records->concession_areas[i]; + concession_areas[it->key] = it; + } + financers.reserve(records->financers.size()); + for (size_t i = 0; i < records->financers.size(); i++) { + auto *it = &records->financers[i]; + financers[it->key] = it; + } + journey_pattern_timing_links.reserve(records->journey_pattern_timing_links.size()); + for (size_t i = 0; i < records->journey_pattern_timing_links.size(); i++) { + auto *it = &records->journey_pattern_timing_links[i]; + journey_pattern_timing_links[it->key] = it; + } + points.reserve(records->points.size()); + for (size_t i = 0; i < records->points.size(); i++) { + auto *it = &records->points[i]; + points[it->key] = it; + } + point_on_links.reserve(records->point_on_links.size()); + for (size_t i = 0; i < records->point_on_links.size(); i++) { + auto *it = &records->point_on_links[i]; + point_on_links[it->key] = it; + } + icons.reserve(records->icons.size()); + for (size_t i = 0; i < records->icons.size(); i++) { + auto *it = &records->icons[i]; + icons[it->key] = it; + } + notices.reserve(records->notices.size()); + for (size_t i = 0; i < records->notices.size(); i++) { + auto *it = &records->notices[i]; + notices[it->key] = it; + } + time_demand_groups.reserve(records->time_demand_groups.size()); + for (size_t i = 0; i < records->time_demand_groups.size(); i++) { + auto *it = &records->time_demand_groups[i]; + time_demand_groups[it->key] = it; + } + time_demand_group_run_times.reserve(records->time_demand_group_run_times.size()); + for (size_t i = 0; i < records->time_demand_group_run_times.size(); i++) { + auto *it = &records->time_demand_group_run_times[i]; + time_demand_group_run_times[it->key] = it; + } + period_groups.reserve(records->period_groups.size()); + for (size_t i = 0; i < records->period_groups.size(); i++) { + auto *it = &records->period_groups[i]; + period_groups[it->key] = it; + } + specific_days.reserve(records->specific_days.size()); + for (size_t i = 0; i < records->specific_days.size(); i++) { + auto *it = &records->specific_days[i]; + specific_days[it->key] = it; + } + timetable_versions.reserve(records->timetable_versions.size()); + for (size_t i = 0; i < records->timetable_versions.size(); i++) { + auto *it = &records->timetable_versions[i]; + timetable_versions[it->key] = it; + } + public_journeys.reserve(records->public_journeys.size()); + for (size_t i = 0; i < records->public_journeys.size(); i++) { + auto *it = &records->public_journeys[i]; + public_journeys[it->key] = it; + } + period_group_validities.reserve(records->period_group_validities.size()); + for (size_t i = 0; i < records->period_group_validities.size(); i++) { + auto *it = &records->period_group_validities[i]; + period_group_validities[it->key] = it; + } + exceptional_operating_days.reserve(records->exceptional_operating_days.size()); + for (size_t i = 0; i < records->exceptional_operating_days.size(); i++) { + auto *it = &records->exceptional_operating_days[i]; + exceptional_operating_days[it->key] = it; + } + schedule_versions.reserve(records->schedule_versions.size()); + for (size_t i = 0; i < records->schedule_versions.size(); i++) { + auto *it = &records->schedule_versions[i]; + schedule_versions[it->key] = it; + } + public_journey_passing_times.reserve(records->public_journey_passing_times.size()); + for (size_t i = 0; i < records->public_journey_passing_times.size(); i++) { + auto *it = &records->public_journey_passing_times[i]; + public_journey_passing_times[it->key] = it; + } + operating_days.reserve(records->operating_days.size()); + for (size_t i = 0; i < records->operating_days.size(); i++) { + auto *it = &records->operating_days[i]; + operating_days[it->key] = it; + } +} + +size_t Kv1Index::size() const { + return organizational_units.size() + + higher_organizational_units.size() + + user_stop_points.size() + + user_stop_areas.size() + + timing_links.size() + + links.size() + + lines.size() + + destinations.size() + + journey_patterns.size() + + concession_financer_relations.size() + + concession_areas.size() + + financers.size() + + journey_pattern_timing_links.size() + + points.size() + + point_on_links.size() + + icons.size() + + notices.size() + + time_demand_groups.size() + + time_demand_group_run_times.size() + + period_groups.size() + + specific_days.size() + + timetable_versions.size() + + public_journeys.size() + + period_group_validities.size() + + exceptional_operating_days.size() + + schedule_versions.size() + + public_journey_passing_times.size() + + operating_days.size(); +} + +void kv1LinkRecords(Kv1Index &index) { + for (auto &orunorun : index.records->higher_organizational_units) { + Kv1OrganizationalUnit::Key orun_parent_key( + orunorun.key.data_owner_code, + orunorun.key.organizational_unit_code_parent); + Kv1OrganizationalUnit::Key orun_child_key( + orunorun.key.data_owner_code, + orunorun.key.organizational_unit_code_child); + orunorun.p_organizational_unit_parent = index.organizational_units[orun_parent_key]; + orunorun.p_organizational_unit_child = index.organizational_units[orun_child_key]; + } + for (auto &usrstop : index.records->user_stop_points) { + Kv1Point::Key point_key( + usrstop.key.data_owner_code, + usrstop.key.user_stop_code); + usrstop.p_point = index.points[point_key]; + if (!usrstop.user_stop_area_code.empty()) { + Kv1UserStopArea::Key usrstar_key( + usrstop.key.data_owner_code, + usrstop.user_stop_area_code); + usrstop.p_user_stop_area = index.user_stop_areas[usrstar_key]; + } + } + for (auto &tili : index.records->timing_links) { + Kv1UserStopPoint::Key usrstop_begin_key( + tili.key.data_owner_code, + tili.key.user_stop_code_begin); + Kv1UserStopPoint::Key usrstop_end_key( + tili.key.data_owner_code, + tili.key.user_stop_code_end); + tili.p_user_stop_begin = index.user_stop_points[usrstop_begin_key]; + tili.p_user_stop_end = index.user_stop_points[usrstop_end_key]; + } + for (auto &link : index.records->links) { + Kv1UserStopPoint::Key usrstop_begin_key( + link.key.data_owner_code, + link.key.user_stop_code_begin); + Kv1UserStopPoint::Key usrstop_end_key( + link.key.data_owner_code, + link.key.user_stop_code_end); + link.p_user_stop_begin = index.user_stop_points[usrstop_begin_key]; + link.p_user_stop_end = index.user_stop_points[usrstop_end_key]; + } + for (auto &line : index.records->lines) { + if (!line.line_icon) + continue; + Kv1Icon::Key icon_key( + line.key.data_owner_code, + *line.line_icon); + line.p_line_icon = index.icons[icon_key]; + } + for (auto &jopa : index.records->journey_patterns) { + Kv1Line::Key line_key( + jopa.key.data_owner_code, + jopa.key.line_planning_number); + jopa.p_line = index.lines[line_key]; + } + for (auto &confinrel : index.records->concession_financer_relations) { + Kv1ConcessionArea::Key conarea_key( + confinrel.key.data_owner_code, + confinrel.concession_area_code); + confinrel.p_concession_area = index.concession_areas[conarea_key]; + if (!confinrel.financer_code.empty()) { + Kv1Financer::Key financer_key( + confinrel.key.data_owner_code, + confinrel.financer_code); + confinrel.p_financer = index.financers[financer_key]; + } + } + for (auto &jopatili : index.records->journey_pattern_timing_links) { + Kv1Line::Key line_key( + jopatili.key.data_owner_code, + jopatili.key.line_planning_number); + Kv1JourneyPattern::Key jopa_key( + jopatili.key.data_owner_code, + jopatili.key.line_planning_number, + jopatili.key.journey_pattern_code); + Kv1UserStopPoint::Key usrstop_begin_key( + jopatili.key.data_owner_code, + jopatili.user_stop_code_begin); + Kv1UserStopPoint::Key usrstop_end_key( + jopatili.key.data_owner_code, + jopatili.user_stop_code_end); + Kv1ConcessionFinancerRelation::Key confinrel_key( + jopatili.key.data_owner_code, + jopatili.con_fin_rel_code); + Kv1Destination::Key dest_key( + jopatili.key.data_owner_code, + jopatili.dest_code); + jopatili.p_line = index.lines[line_key]; + jopatili.p_journey_pattern = index.journey_patterns[jopa_key]; + jopatili.p_user_stop_begin = index.user_stop_points[usrstop_begin_key]; + jopatili.p_user_stop_end = index.user_stop_points[usrstop_end_key]; + jopatili.p_con_fin_rel = index.concession_financer_relations[confinrel_key]; + jopatili.p_dest = index.destinations[dest_key]; + if (jopatili.line_dest_icon) { + Kv1Icon::Key icon_key{ + jopatili.key.data_owner_code, + *jopatili.line_dest_icon, + }; + jopatili.p_line_dest_icon = index.icons[icon_key]; + } + } + for (auto &pool : index.records->point_on_links) { + Kv1UserStopPoint::Key usrstop_begin_key( + pool.key.data_owner_code, + pool.key.user_stop_code_begin); + Kv1UserStopPoint::Key usrstop_end_key( + pool.key.data_owner_code, + pool.key.user_stop_code_end); + Kv1Point::Key point_key( + pool.key.point_data_owner_code, + pool.key.point_code); + pool.p_user_stop_begin = index.user_stop_points[usrstop_begin_key]; + pool.p_user_stop_end = index.user_stop_points[usrstop_end_key]; + pool.p_point = index.points[point_key]; + } + for (auto &ntcassgnm : index.records->notice_assignments) { + Kv1Notice::Key notice_key( + ntcassgnm.data_owner_code, + ntcassgnm.notice_code); + ntcassgnm.p_notice = index.notices[notice_key]; + } + for (auto &timdemgrp : index.records->time_demand_groups) { + Kv1Line::Key line_key( + timdemgrp.key.data_owner_code, + timdemgrp.key.line_planning_number); + Kv1JourneyPattern::Key jopa_key( + timdemgrp.key.data_owner_code, + timdemgrp.key.line_planning_number, + timdemgrp.key.journey_pattern_code); + timdemgrp.p_line = index.lines[line_key]; + timdemgrp.p_journey_pattern = index.journey_patterns[jopa_key]; + } + for (auto &timdemrnt : index.records->time_demand_group_run_times) { + Kv1Line::Key line_key( + timdemrnt.key.data_owner_code, + timdemrnt.key.line_planning_number); + Kv1JourneyPattern::Key jopa_key( + timdemrnt.key.data_owner_code, + timdemrnt.key.line_planning_number, + timdemrnt.key.journey_pattern_code); + Kv1TimeDemandGroup::Key timdemgrp_key( + timdemrnt.key.data_owner_code, + timdemrnt.key.line_planning_number, + timdemrnt.key.journey_pattern_code, + timdemrnt.key.time_demand_group_code); + Kv1UserStopPoint::Key usrstop_begin_key( + timdemrnt.key.data_owner_code, + timdemrnt.user_stop_code_begin); + Kv1UserStopPoint::Key usrstop_end_key( + timdemrnt.key.data_owner_code, + timdemrnt.user_stop_code_end); + Kv1JourneyPatternTimingLink::Key jopatili_key( + timdemrnt.key.data_owner_code, + timdemrnt.key.line_planning_number, + timdemrnt.key.journey_pattern_code, + timdemrnt.key.timing_link_order); + timdemrnt.p_line = index.lines[line_key]; + timdemrnt.p_user_stop_end = index.user_stop_points[usrstop_end_key]; + timdemrnt.p_user_stop_begin = index.user_stop_points[usrstop_begin_key]; + timdemrnt.p_journey_pattern = index.journey_patterns[jopa_key]; + timdemrnt.p_time_demand_group = index.time_demand_groups[timdemgrp_key]; + timdemrnt.p_journey_pattern_timing_link = index.journey_pattern_timing_links[jopatili_key]; + } + for (auto &tive : index.records->timetable_versions) { + Kv1OrganizationalUnit::Key orun_key( + tive.key.data_owner_code, + tive.key.organizational_unit_code); + Kv1PeriodGroup::Key pegr_key( + tive.key.data_owner_code, + tive.key.period_group_code); + Kv1SpecificDay::Key specday_key( + tive.key.data_owner_code, + tive.key.specific_day_code); + tive.p_organizational_unit = index.organizational_units[orun_key]; + tive.p_period_group = index.period_groups[pegr_key]; + tive.p_specific_day = index.specific_days[specday_key]; + } + for (auto &pujo : index.records->public_journeys) { + Kv1TimetableVersion::Key tive_key( + pujo.key.data_owner_code, + pujo.key.organizational_unit_code, + pujo.key.timetable_version_code, + pujo.key.period_group_code, + pujo.key.specific_day_code); + Kv1OrganizationalUnit::Key orun_key( + pujo.key.data_owner_code, + pujo.key.organizational_unit_code); + Kv1PeriodGroup::Key pegr_key( + pujo.key.data_owner_code, + pujo.key.period_group_code); + Kv1SpecificDay::Key specday_key( + pujo.key.data_owner_code, + pujo.key.specific_day_code); + Kv1Line::Key line_key( + pujo.key.data_owner_code, + pujo.key.line_planning_number); + Kv1TimeDemandGroup::Key timdemgrp_key( + pujo.key.data_owner_code, + pujo.key.line_planning_number, + pujo.journey_pattern_code, + pujo.time_demand_group_code); + Kv1JourneyPattern::Key jopa_key( + pujo.key.data_owner_code, + pujo.key.line_planning_number, + pujo.journey_pattern_code); + pujo.p_timetable_version = index.timetable_versions[tive_key]; + pujo.p_organizational_unit = index.organizational_units[orun_key]; + pujo.p_period_group = index.period_groups[pegr_key]; + pujo.p_specific_day = index.specific_days[specday_key]; + pujo.p_line = index.lines[line_key]; + pujo.p_time_demand_group = index.time_demand_groups[timdemgrp_key]; + pujo.p_journey_pattern = index.journey_patterns[jopa_key]; + } + for (auto &pegrval : index.records->period_group_validities) { + Kv1OrganizationalUnit::Key orun_key( + pegrval.key.data_owner_code, + pegrval.key.organizational_unit_code); + Kv1PeriodGroup::Key pegr_key( + pegrval.key.data_owner_code, + pegrval.key.period_group_code); + pegrval.p_organizational_unit = index.organizational_units[orun_key]; + pegrval.p_period_group = index.period_groups[pegr_key]; + } + for (auto &excopday : index.records->exceptional_operating_days) { + Kv1OrganizationalUnit::Key orun_key( + excopday.key.data_owner_code, + excopday.key.organizational_unit_code); + Kv1SpecificDay::Key specday_key( + excopday.key.data_owner_code, + excopday.specific_day_code); + Kv1PeriodGroup::Key pegr_key( + excopday.key.data_owner_code, + excopday.period_group_code); + excopday.p_organizational_unit = index.organizational_units[orun_key]; + excopday.p_specific_day = index.specific_days[specday_key]; + excopday.p_period_group = index.period_groups[pegr_key]; + } + for (auto &schedvers : index.records->schedule_versions) { + Kv1OrganizationalUnit::Key orun_key( + schedvers.key.data_owner_code, + schedvers.key.organizational_unit_code); + schedvers.p_organizational_unit = index.organizational_units[orun_key]; + } + for (auto &pujopass : index.records->public_journey_passing_times) { + Kv1OrganizationalUnit::Key orun_key( + pujopass.key.data_owner_code, + pujopass.key.organizational_unit_code); + Kv1ScheduleVersion::Key schedvers_key( + pujopass.key.data_owner_code, + pujopass.key.organizational_unit_code, + pujopass.key.schedule_code, + pujopass.key.schedule_type_code); + Kv1Line::Key line_key( + pujopass.key.data_owner_code, + pujopass.key.line_planning_number); + Kv1JourneyPattern::Key jopa_key( + pujopass.key.data_owner_code, + pujopass.key.line_planning_number, + pujopass.journey_pattern_code); + Kv1UserStopPoint::Key usrstop_key( + pujopass.key.data_owner_code, + pujopass.user_stop_code); + pujopass.p_organizational_unit = index.organizational_units[orun_key]; + pujopass.p_schedule_version = index.schedule_versions[schedvers_key]; + pujopass.p_line = index.lines[line_key]; + pujopass.p_journey_pattern = index.journey_patterns[jopa_key]; + pujopass.p_user_stop = index.user_stop_points[usrstop_key]; + } + for (auto &operday : index.records->operating_days) { + Kv1OrganizationalUnit::Key orun_key( + operday.key.data_owner_code, + operday.key.organizational_unit_code); + Kv1ScheduleVersion::Key schedvers_key( + operday.key.data_owner_code, + operday.key.organizational_unit_code, + operday.key.schedule_code, + operday.key.schedule_type_code); + operday.p_organizational_unit = index.organizational_units[orun_key]; + operday.p_schedule_version = index.schedule_versions[schedvers_key]; + } +} diff --git a/lib/libtmi8/src/kv1_lexer.cpp b/lib/libtmi8/src/kv1_lexer.cpp new file mode 100644 index 0000000..028127b --- /dev/null +++ b/lib/libtmi8/src/kv1_lexer.cpp @@ -0,0 +1,152 @@ +// vim:set sw=2 ts=2 sts et: + +#include + +Kv1Lexer::Kv1Lexer(std::string_view input) + : input(input), slice(input) +{} + +// Does not eat newline character. +void Kv1Lexer::eatRestOfLine() { + size_t end = slice.size(); + for (size_t i = 0; i < slice.size(); i++) { + if (slice[i] == '\r' || slice[i] == '\n') { + end = i; + break; + } + } + slice = slice.substr(end); +} + +void Kv1Lexer::lexOptionalHeader() { + if (slice.starts_with('[')) eatRestOfLine(); +} + +void Kv1Lexer::lexOptionalComment() { + if (slice.starts_with(';')) eatRestOfLine(); +} + +inline bool Kv1Lexer::isWhitespace(int c) { + return c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\v'; +} + +void Kv1Lexer::readQuotedColumn() { + Kv1Token token{ .type = KV1_TOKEN_CELL }; + + if (slice.size() == 0 || slice[0] != '"') { + errors.push_back("(internal error) readQuotedColumn: slice[0] != '\"'"); + return; + } + slice = slice.substr(1); + while (true) { + size_t quote = slice.find('"'); + if (quote == std::string_view::npos) { + errors.push_back("readQuotedColumn: no matching closing quote found"); + return; + } + if (quote+1 == slice.size() || slice[quote + 1] != '"') { + token.data.append(slice.substr(0, quote)); + break; + } + token.data.append(slice.substr(0, quote + 1)); + slice = slice.substr(quote + 2); + } + + size_t end = slice.size(); + for (size_t i = 0; i < slice.size(); i++) { + if (slice[i] == '|' || slice[i] == '\r' || slice[i] == '\n') { + end = i; + break; + } + if (!isWhitespace(slice[i])) { + errors.push_back("readQuotedColumn: encountered non-whitespace character after closing quote"); + return; + } + } + if (end != std::string_view::npos) slice = slice.substr(end); + else slice = slice.substr(slice.size()); + + tokens.push_back(std::move(token)); +} + +void Kv1Lexer::readUnquotedColumn() { + size_t end = slice.size(); + size_t content_end = 0; + for (size_t i = 0; i < slice.size(); i++) { + if (slice[i] == '|' || slice[i] == '\r' || slice[i] == '\n') { + end = i; + break; + } else if (!isWhitespace(slice[i])) { + content_end = i + 1; + } + } + tokens.emplace_back(KV1_TOKEN_CELL, std::string(slice.substr(0, content_end))); + if (end != std::string_view::npos) slice = slice.substr(end); + else slice = slice.substr(slice.size()); +} + +void Kv1Lexer::lexRow() { + size_t cols = 0; + while (slice.size() > 0 && slice[0] != '\r' && slice[0] != '\n') { + if (slice[0] == '"') readQuotedColumn(); + else readUnquotedColumn(); + if (!errors.empty()) return; + cols++; + if (slice.size() != 0) { + if (slice[0] == '|') { + slice = slice.substr(1); + // A newline/eof right after pipe? That means an empty field at the end + // of the record, we also want to emit that as a token. + if (slice.size() == 0 || slice[0] == '\r' || slice[0] == '\n') { + tokens.push_back({ .type = KV1_TOKEN_CELL }); + } + } else if (slice[0] == '\r') { + if (slice.size() > 1 && slice[1] == '\n') slice = slice.substr(2); + else slice = slice.substr(1); + break; + } else if (slice[0] == '\n') { + slice = slice.substr(1); + break; + } else { + errors.push_back("lexRow: expected CR, LF or |"); + return; + } + } + } + tokens.push_back({ .type = KV1_TOKEN_ROW_END }); +} + +// Returns true when a line ending was consumed. +bool Kv1Lexer::eatWhitespace() { + for (size_t i = 0; i < slice.size(); i++) { + if (slice[i] == '\r') { + slice = slice.substr(i + 1); + if (slice.size() > 1 && slice[i + 1] == '\n') + slice = slice.substr(i + 2); + return true; + } + if (slice[i] == '\n') { + slice = slice.substr(i + 1); + return true; + } + + if (slice[i] != ' ' && slice[i] != '\f' && slice[i] != '\t' && slice[i] != '\v') { + slice = slice.substr(i); + return false; + } + } + return false; +} + +void Kv1Lexer::lex() { + lexOptionalHeader(); + eatWhitespace(); + + while (errors.empty() && !slice.empty()) { + lexOptionalComment(); + bool newline = eatWhitespace(); + if (newline) continue; + // We are now either (1) at the end of the file or (2) at the start of some column data + if (errors.empty()) lexRow(); + } +} diff --git a/lib/libtmi8/src/kv1_parser.cpp b/lib/libtmi8/src/kv1_parser.cpp new file mode 100644 index 0000000..ac0c6bf --- /dev/null +++ b/lib/libtmi8/src/kv1_parser.cpp @@ -0,0 +1,1258 @@ +// vim:set sw=2 ts=2 sts et: + +#include + +using rune = uint32_t; + +static size_t decodeUtf8Cp(std::string_view s, rune *dest = nullptr) { + rune res = 0xFFFD; + size_t length = 1; + + if (s.size() == 0) + return 0; + const uint8_t *b = reinterpret_cast(s.data()); + if (!(b[0] & 0x80)) + res = static_cast(b[0]); + else if ((b[0] & 0xE0) == 0xC0) { + length = 2; + if (s.size() >= 2 && (b[1] & 0xC0) == 0x80) { + res = static_cast(b[0] & ~0xC0) << 6; + res |= static_cast(b[1] & ~0x80); + } + } else if ((b[0] & 0xF0) == 0xE0) { + length = 3; + if (s.size() >= 3 && (b[1] & 0xC0) == 0x80 && (b[2] & 0xC0) == 0x80) { + res = static_cast(b[0] & ~0xE0) << 12; + res |= static_cast(b[1] & ~0x80) << 6; + res |= static_cast(b[2] & ~0x80); + } + } else if (b[0] == 0xF0) { + length = 4; + if (s.size() >= 4 && (b[1] & 0xC0) == 0x80 && (b[2] & 0xC0) == 0x80 && (b[3] & 0xC0) == 0x80) { + res = static_cast(b[0] & ~0xF0) << 18; + res |= static_cast(b[1] & ~0x80) << 12; + res |= static_cast(b[2] & ~0x80) << 6; + res |= static_cast(b[3] & ~0x80); + } + } + + if (dest) + *dest = res; + return length; +} + +// Counts the number of codepoints in a valid UTF-8 string. Returns SIZE_MAX if +// the string contains invalid UTF-8 codepoints. +static size_t stringViewLengthUtf8(std::string_view sv) { + size_t codepoints = 0; + while (sv.size() > 0) { + size_t codepoint_size = decodeUtf8Cp(sv); + if (codepoint_size == 0) return SIZE_MAX; + codepoints++; + sv = sv.substr(codepoint_size); + } + return codepoints; +} + +Kv1Parser::Kv1Parser(std::vector tokens, Kv1Records &parse_into) + : tokens(std::move(tokens)), + records(parse_into) +{} + +bool Kv1Parser::atEnd() const { + return pos >= tokens.size(); +} + +void Kv1Parser::eatRowEnds() { + while (!atEnd() && tokens[pos].type == KV1_TOKEN_ROW_END) pos++; +} + +const Kv1Token *Kv1Parser::cur() const { + if (atEnd()) return nullptr; + return &tokens[pos]; +} + +const std::string *Kv1Parser::eatCell(std::string_view parsing_what) { + const Kv1Token *tok = cur(); + if (!tok) { + record_errors.push_back(std::format("Expected cell but got end of file when parsing {}", parsing_what)); + return nullptr; + } + if (tok->type == KV1_TOKEN_ROW_END) { + record_errors.push_back(std::format("Expected cell but got end of row when parsing {}", parsing_what)); + return nullptr; + } + pos++; + return &tok->data; +} + +void Kv1Parser::requireString(std::string_view field, bool mandatory, size_t max_length, std::string_view value) { + if (value.empty() && mandatory) { + record_errors.push_back(std::format("{} has length zero but is required", field)); + return; + } + size_t codepoints = stringViewLengthUtf8(value); + if (codepoints == SIZE_MAX) { + global_errors.push_back(std::format("{} contains invalid UTF-8 code points", field)); + return; + } + if (codepoints > max_length) { + record_errors.push_back(std::format("{} has length ({}) that is greater than maximum length ({})", + field, value.size(), max_length)); + } +} + +static inline std::optional parseBoolean(std::string_view src) { + if (src == "1") return true; + if (src == "0") return false; + if (src == "true") return true; + if (src == "false") return false; + return std::nullopt; +} + +std::optional Kv1Parser::requireBoolean(std::string_view field, bool mandatory, std::string_view value) { + if (value.empty()) { + if (mandatory) + record_errors.push_back(std::format("{} is required, but has no value", field)); + return std::nullopt; + } + auto parsed = parseBoolean(value); + if (!parsed.has_value()) + record_errors.push_back(std::format("{} should have value \"1\", \"0\", \"true\" or \"false\"", field)); + return parsed; +} + +static inline size_t countDigits(long x) { + size_t digits = 0; + while (x != 0) { digits++; x /= 10; } + return digits; +} + +std::optional Kv1Parser::requireNumber(std::string_view field, bool mandatory, size_t max_digits, std::string_view value) { + if (value.empty()) { + if (mandatory) + record_errors.push_back(std::format("{} has no value but is required", field)); + return std::nullopt; + } + + double parsed; + auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed, std::chars_format::fixed); + if (ec != std::errc()) { + record_errors.push_back(std::format("{} has a bad value that cannot be parsed as a number", field)); + return std::nullopt; + } + if (ptr != value.data() + value.size()) { + record_errors.push_back(std::format("{} contains characters that were not parsed as a number", field)); + return std::nullopt; + } + + size_t digits = countDigits(static_cast(parsed)); + if (digits > max_digits) { + record_errors.push_back(std::format("{} contains more digits (in the integral part) ({}) than allowed ({})", + field, digits, max_digits)); + return std::nullopt; + } + + return parsed; +} + +static inline bool isHexDigit(char c) { + return (c >= '0' && c <= '9') || (c >= 'A' && c <= 'F'); +} + +static inline uint8_t fromHex(char c) { + if (c >= '0' && c <= '9') return static_cast(c - '0'); + else if (c >= 'A' && c <= 'F') return static_cast(c - 'A' + 10); + return 0; +} + +static std::optional parseRgbColor(std::string_view src) { + bool valid = src.size() == 6 + && isHexDigit(src[0]) && isHexDigit(src[1]) + && isHexDigit(src[2]) && isHexDigit(src[3]) + && isHexDigit(src[4]) && isHexDigit(src[5]); + if (!valid) return std::nullopt; + uint8_t r = static_cast(fromHex(src[0]) << 4) + fromHex(src[1]); + uint8_t g = static_cast(fromHex(src[2]) << 4) + fromHex(src[3]); + uint8_t b = static_cast(fromHex(src[4]) << 4) + fromHex(src[5]); + return RgbColor{ r, g, b }; +} + +std::optional Kv1Parser::requireRgbColor(std::string_view field, bool mandatory, std::string_view value) { + if (value.empty()) { + if (mandatory) + record_errors.push_back(std::format("{} is required, but has no value", field)); + return std::nullopt; + } + auto parsed = parseRgbColor(value); + if (!parsed.has_value()) + record_errors.push_back(std::format("{} should be an RGB color, i.e. a sequence of six hexadecimally represented nibbles", field)); + return parsed; +} + +std::optional Kv1Parser::requireRdCoord(std::string_view field, bool mandatory, size_t min_digits, std::string_view value) { + if (value.empty()) { + if (mandatory) + record_errors.push_back(std::format("{} is required, but has no value", field)); + return std::nullopt; + } + if (value.size() > 15) { + record_errors.push_back(std::format("{} may not have more than 15 characters", field)); + return std::nullopt; + } + + double parsed; + auto [ptr, ec] = std::from_chars(value.data(), value.data() + value.size(), parsed, std::chars_format::fixed); + if (ec != std::errc()) { + record_errors.push_back(std::format("{} has a bad value that cannot be parsed as a number", field)); + return std::nullopt; + } + if (ptr != value.data() + value.size()) { + record_errors.push_back(std::format("{} contains characters that were not parsed as a number", field)); + return std::nullopt; + } + + size_t digits = countDigits(static_cast(parsed)); + if (digits < min_digits) { + record_errors.push_back(std::format("{} contains less digits (in the integral part) ({}) than required ({}) [value: {}]", + field, digits, min_digits, value)); + return std::nullopt; + } + + return parsed; +} + +std::string Kv1Parser::eatString(std::string_view field, bool mandatory, size_t max_length) { + auto value = eatCell(field); + if (!record_errors.empty()) return {}; + requireString(field, mandatory, max_length, *value); + return std::move(*value); +} + +std::optional Kv1Parser::eatBoolean(std::string_view field, bool mandatory) { + auto value = eatCell(field); + if (!record_errors.empty()) return {}; + return requireBoolean(field, mandatory, *value); +} + +std::optional Kv1Parser::eatNumber(std::string_view field, bool mandatory, size_t max_digits) { + auto value = eatCell(field); + if (!record_errors.empty()) return {}; + return requireNumber(field, mandatory, max_digits, *value); +} + +std::optional Kv1Parser::eatRgbColor(std::string_view field, bool mandatory) { + auto value = eatCell(field); + if (!record_errors.empty()) return {}; + return requireRgbColor(field, mandatory, *value); +} + +std::optional Kv1Parser::eatRdCoord(std::string_view field, bool mandatory, size_t min_digits) { + auto value = eatCell(field); + if (!record_errors.empty()) return {}; + return requireRdCoord(field, mandatory, min_digits, *value); +} + +std::string Kv1Parser::parseHeader() { + auto record_type = eatString("
.Recordtype", true, 10); + auto version_number = eatString("
.VersionNumber", true, 2); + auto implicit_explicit = eatString("
.Implicit/Explicit", true, 1); + if (!record_errors.empty()) return {}; + + if (version_number != "1") { + record_errors.push_back("
.VersionNumber should be 1"); + return ""; + } + if (implicit_explicit != "I") { + record_errors.push_back("
.Implicit/Explicit should be 'I'"); + return ""; + } + + return record_type; +} + +void Kv1Parser::eatRestOfRow() { + while (!atEnd() && cur()->type != KV1_TOKEN_ROW_END) pos++; +} + +void Kv1Parser::parse() { + while (!atEnd()) { + eatRowEnds(); + if (atEnd()) return; + + std::string record_type = parseHeader(); + if (!record_errors.empty()) break; + if (!type_parsers.contains(record_type)) { + warns.push_back(std::format("Recordtype ({}) is bad or names a record type that this program cannot process", + record_type)); + eatRestOfRow(); + continue; + } + + ParseFunc parseType = Kv1Parser::type_parsers.at(record_type); + (this->*parseType)(); + if (cur() && cur()->type != KV1_TOKEN_ROW_END) { + record_errors.push_back(std::format("Parser function for Recordtype ({}) did not eat all record fields", + record_type)); + eatRestOfRow(); + } + if (!record_errors.empty()) { + global_errors.insert(global_errors.end(), record_errors.begin(), record_errors.end()); + record_errors.clear(); + } + } +} + +void Kv1Parser::parseOrganizationalUnit() { + auto data_owner_code = eatString("ORUN.DataOwnerCode", true, 10); + auto organizational_unit_code = eatString("ORUN.OrganizationalUnitCode", true, 10); + auto name = eatString("ORUN.Name", true, 50); + auto organizational_unit_type = eatString("ORUN.OrganizationalUnitType", true, 10); + auto description = eatString("ORUN.Description", false, 255); + if (!record_errors.empty()) return; + + records.organizational_units.emplace_back( + Kv1OrganizationalUnit::Key( + data_owner_code, + organizational_unit_code), + name, + organizational_unit_type, + description); +} + +static inline bool isDigit(char c) { + return c >= '0' && c <= '9'; +} + +// Parse a string of the format YYYY-MM-DD. +static std::optional parseYyyymmdd(std::string_view src) { + bool valid = src.size() == 10 + && isDigit(src[0]) && isDigit(src[1]) + && isDigit(src[2]) && isDigit(src[3]) && src[4] == '-' + && isDigit(src[5]) && isDigit(src[6]) && src[7] == '-' + && isDigit(src[8]) && isDigit(src[9]); + if (!valid) return std::nullopt; + int year = (src[0] - '0') * 1000 + (src[1] - '0') * 100 + (src[2] - '0') * 10 + src[3] - '0'; + int month = (src[5] - '0') * 10 + src[6] - '0'; + int day = (src[8] - '0') * 10 + src[9] - '0'; + return std::chrono::year(year) / std::chrono::month(month) / std::chrono::day(day); +} + +// Parse a string of the format HH:MM:SS. +static std::optional> parseHhmmss(std::string_view src) { + bool valid = src.size() == 8 + && isDigit(src[0]) && isDigit(src[1]) && src[2] == ':' + && isDigit(src[3]) && isDigit(src[4]) && src[5] == ':' + && isDigit(src[6]) && isDigit(src[7]); + if (!valid) return std::nullopt; + int hh = (src[0] - '0') * 10 + src[1] - '0'; + int mm = (src[3] - '0') * 10 + src[4] - '0'; + int ss = (src[6] - '0') * 10 + src[7] - '0'; + // The check for the hour not being greater than 32 comes from the fact the + // specification explicitly allows hours greater than 23, noting that the + // period 24:00-32:00 is equivalent to 00:00-08:00 in the next day, for + // exploitation of two days. + if (hh > 32 || mm > 59 || ss > 59) return std::nullopt; + return std::chrono::hh_mm_ss(std::chrono::hours(hh) + std::chrono::minutes(mm) + std::chrono::seconds(ss)); +} + +static std::optional parseDateTime(std::string_view src, const std::chrono::time_zone *amsterdam, std::string_view *error = nullptr) { +#define ERROR(err) do { if (error) *error = err; return std::nullopt; } while (0) + if (src.size() > 23) ERROR("timestamp string is too big"); + if (src.size() < 17) ERROR("timestamp string is too small"); + + bool valid_year = isDigit(src[0]) && isDigit(src[1]) && isDigit(src[2]) && isDigit(src[3]); + if (!valid_year) ERROR("year has bad format"); + + size_t month_off = src[4] == '-' ? 5 : 4; + size_t day_off = src[month_off + 2] == '-' ? month_off + 3 : month_off + 2; + size_t time_off = day_off + 2; + if (src[time_off] != 'T' && src[time_off] != ' ') + ERROR("missing date/time separator"); + size_t tzd_off = time_off + 9; + // For clarity, TZD stands for Time Zone Designator. It often takes the form + // of Z (Zulu, UTC+00:00) or as an offset from UTC in hours and minutes, + // formatted as +|-HH:MM (e.g. +01:00, -12:00). + + if (time_off + 8 >= src.size()) ERROR("bad format, not enough space for hh:mm:ss"); + + int year = (src[0] - '0') * 1000 + (src[1] - '0') * 100 + (src[2] - '0') * 10 + src[3] - '0'; + int month = (src[month_off] - '0') * 10 + src[month_off + 1] - '0'; + int day = (src[day_off] - '0') * 10 + src[day_off + 1] - '0'; + int hour = (src[time_off + 1] - '0') * 10 + src[time_off + 2] - '0'; + int minute = (src[time_off + 4] - '0') * 10 + src[time_off + 5] - '0'; + int second = (src[time_off + 7] - '0') * 10 + src[time_off + 8] - '0'; + + auto date = std::chrono::year(year) / std::chrono::month(month) / std::chrono::day(day); + auto time = std::chrono::hours(hour) + std::chrono::minutes(minute) + std::chrono::seconds(second); + + std::chrono::sys_seconds unix_start_of_day; + if (tzd_off < src.size()) { + unix_start_of_day = std::chrono::sys_days(date); + } else { + auto local_days = std::chrono::local_days(date); + std::chrono::zoned_seconds zoned_start_of_day = std::chrono::zoned_time(amsterdam, local_days); + unix_start_of_day = std::chrono::sys_seconds(zoned_start_of_day); + } + + std::chrono::minutes offset(0); + if (tzd_off + 1 == src.size() && src[tzd_off] != 'Z') { + ERROR("bad TZD (missing Zulu indicator)"); + } else if (tzd_off + 6 == src.size()) { + bool valid_tzd = (src[tzd_off] == '+' || src[tzd_off] == '-') + && isDigit(src[tzd_off + 1]) && isDigit(src[tzd_off + 2]) && src[tzd_off + 3] == ':' + && isDigit(src[tzd_off + 4]) && isDigit(src[tzd_off + 5]); + if (!valid_tzd) ERROR("bad offset TZD format (expected +|-hh:mm)"); + int sign = src[tzd_off] == '-' ? -1 : 1; + int tzd_hh = (src[tzd_off + 1] - '0') * 10 + src[tzd_off + 2] - '0'; + int tzd_mm = (src[tzd_off + 3] - '0') * 10 + src[tzd_off + 4] - '0'; + offset = sign * std::chrono::minutes(tzd_hh * 60 + tzd_mm); + } else if (tzd_off < src.size()) { + // There is a TZD but we literally have no clue how to parse it :/ + ERROR("cannot parse TZD of unexpected length"); + } + + return unix_start_of_day + time - offset; +#undef ERROR +} + +void Kv1Parser::parseHigherOrganizationalUnit() { + auto data_owner_code = eatString("ORUNORUN.DataOwnerCode", true, 10); + auto organizational_unit_code_parent = eatString("ORUNORUN.OrganizationalUnitCodeParent", true, 10); + auto organizational_unit_code_child = eatString("ORUNORUN.OrganizationalUnitCodeChild", true, 10); + auto valid_from_raw = eatString("ORUNORUN.ValidFrom", true, 10); + if (!record_errors.empty()) return; + + auto valid_from = parseYyyymmdd(valid_from_raw); + if (!valid_from) { + record_errors.push_back("ORUNORUN.ValidFrom has invalid format, should be YYYY-MM-DD"); + return; + } + + records.higher_organizational_units.emplace_back( + Kv1HigherOrganizationalUnit::Key( + data_owner_code, + organizational_unit_code_parent, + organizational_unit_code_child, + *valid_from)); +} + +void Kv1Parser::parseUserStopPoint() { + auto data_owner_code = eatString ("USRSTOP.DataOwnerCode", true, 10); + auto user_stop_code = eatString ("USRSTOP.UserStopCode", true, 10); + auto timing_point_code = eatString ("USRSTOP.TimingPointCode", false, 10); + auto get_in = eatBoolean("USRSTOP.GetIn", true ); + auto get_out = eatBoolean("USRSTOP.GetOut", true ); + eatCell ("USRSTOP." ); + auto name = eatString ("USRSTOP.Name", true, 50); + auto town = eatString ("USRSTOP.Town", true, 50); + auto user_stop_area_code = eatString ("USRSTOP.UserStopAreaCode", false, 10); + auto stop_side_code = eatString ("USRSTOP.StopSideCode", true, 10); + eatCell ("USRSTOP." ); + eatCell ("USRSTOP." ); + auto minimal_stop_time = eatNumber ("USRSTOP.MinimalStopTime", true, 5); + auto stop_side_length = eatNumber ("USRSTOP.StopSideLength", false, 3); + auto description = eatString ("USRSTOP.Description", false, 255); + auto user_stop_type = eatString ("USRSTOP.UserStopType", true, 10); + auto quay_code = eatString ("USRSTOP.QuayCode", false, 30); + if (!record_errors.empty()) return; + + records.user_stop_points.emplace_back( + Kv1UserStopPoint::Key( + data_owner_code, + user_stop_code), + timing_point_code, + *get_in, + *get_out, + name, + town, + user_stop_area_code, + stop_side_code, + *minimal_stop_time, + stop_side_length, + description, + user_stop_type, + quay_code); +} + +void Kv1Parser::parseUserStopArea() { + auto data_owner_code = eatString("USRSTAR.DataOwnerCode", true, 10); + auto user_stop_area_code = eatString("USRSTAR.UserStopAreaCode", true, 10); + auto name = eatString("USRSTAR.Name", true, 50); + auto town = eatString("USRSTAR.Town", true, 50); + eatCell ("USRSTAR." ); + eatCell ("USRSTAR." ); + auto description = eatString("USRSTAR.Description", false, 255); + if (!record_errors.empty()) return; + + records.user_stop_areas.emplace_back( + Kv1UserStopArea::Key( + data_owner_code, + user_stop_area_code), + name, + town, + description); +} + +void Kv1Parser::parseTimingLink() { + auto data_owner_code = eatString("TILI.DataOwnerCode", true, 10); + auto user_stop_code_begin = eatString("TILI.UserStopCodeBegin", true, 10); + auto user_stop_code_end = eatString("TILI.UserStopCodeEnd", true, 10); + auto minimal_drive_time = eatNumber("TILI.MinimalDriveTime", false, 5); + auto description = eatString("TILI.Description", false, 255); + if (!record_errors.empty()) return; + + records.timing_links.emplace_back( + Kv1TimingLink::Key( + data_owner_code, + user_stop_code_begin, + user_stop_code_end), + minimal_drive_time, + description); +} + +void Kv1Parser::parseLink() { + auto data_owner_code = eatString("LINK.DataOwnerCode", true, 10); + auto user_stop_code_begin = eatString("LINK.UserStopCodeBegin", true, 10); + auto user_stop_code_end = eatString("LINK.UserStopCodeEnd", true, 10); + eatCell("LINK." ); + auto distance = eatNumber("LINK.Distance", true, 6); + auto description = eatString("LINK.Description", false, 255); + auto transport_type = eatString("LINK.TransportType", true, 5); + if (!record_errors.empty()) return; + + records.links.emplace_back( + Kv1Link::Key( + data_owner_code, + user_stop_code_begin, + user_stop_code_end, + transport_type), + *distance, + description); +} + +void Kv1Parser::parseLine() { + auto data_owner_code = eatString ("LINE.DataOwnerCode", true, 10); + auto line_planning_number = eatString ("LINE.LinePlanningNumber", true, 10); + auto line_public_number = eatString ("LINE.LinePublicNumber", true, 4); + auto line_name = eatString ("LINE.LineName", true, 50); + auto line_ve_tag_number = eatNumber ("LINE.LineVeTagNumber", true, 3); + auto description = eatString ("LINE.Description", false, 255); + auto transport_type = eatString ("LINE.TransportType", true, 5); + auto line_icon = eatNumber ("LINE.LineIcon", false, 4); + auto line_color = eatRgbColor("LINE.LineColor", false ); + auto line_text_color = eatRgbColor("LINE.LineTextColor", false ); + if (!record_errors.empty()) return; + + // NOTE: This check, although it should be performed to comply with the + // specification, is not actually honored by transit operators (such as + // Connexxion) :/ That's enough reason to keep it disabled here for now. + // if (*line_ve_tag_number < 0 || *line_ve_tag_number > 399) { + // record_errors.push_back(std::format("LINE.LineVeTagNumber is out of range [0-399] with value {}", *line_ve_tag_number)); + // return; + // } + if (*line_ve_tag_number != static_cast(*line_ve_tag_number)) + record_errors.push_back("LINE.LineVeTagNumber should be an integer"); + if (line_icon && *line_icon != static_cast(*line_icon)) + record_errors.push_back("LINE.LineIcon should be an integer"); + if (!record_errors.empty()) return; + + records.lines.emplace_back( + Kv1Line::Key( + data_owner_code, + line_planning_number), + line_public_number, + line_name, + static_cast(*line_ve_tag_number), + description, + transport_type, + static_cast>(line_icon), + line_color, + line_text_color); +} + +void Kv1Parser::parseDestination() { + auto data_owner_code = eatString ("DEST.DataOwnerCode", true, 10); + auto dest_code = eatString ("DEST.DestCode", true, 10); + auto dest_name_full = eatString ("DEST.DestNameFull", true, 50); + auto dest_name_main = eatString ("DEST.DestNameMain", true, 24); + auto dest_name_detail = eatString ("DEST.DestNameDetail", false, 24); + auto relevant_dest_name_detail = eatBoolean ("DEST.RelevantDestNameDetail", true ); + auto dest_name_main_21 = eatString ("DEST.DestNameMain21", true, 21); + auto dest_name_detail_21 = eatString ("DEST.DestNameDetail21", false, 21); + auto dest_name_main_19 = eatString ("DEST.DestNameMain19", true, 19); + auto dest_name_detail_19 = eatString ("DEST.DestNameDetail19", false, 19); + auto dest_name_main_16 = eatString ("DEST.DestNameMain16", true, 16); + auto dest_name_detail_16 = eatString ("DEST.DestNameDetail16", false, 16); + auto dest_icon = eatNumber ("DEST.DestIcon", false, 4); + auto dest_color = eatRgbColor("DEST.DestColor", false ); + // NOTE: Deviating from the offical KV1 specification here. It specifies that + // the maximum length for this field should be 30, but then proceeds to + // specify that it should contain a RGB value comprising of three + // hexadecimally encoded octets, i.e. six characters. We assume that the + // latter is correct and the intended interpretation. + auto dest_text_color = eatRgbColor("DEST.DestTextColor", false ); + if (!record_errors.empty()) return; + + if (dest_icon && *dest_icon != static_cast(*dest_icon)) { + record_errors.push_back("DEST.DestIcon should be an integer"); + return; + } + + records.destinations.emplace_back( + Kv1Destination::Key( + data_owner_code, + dest_code), + dest_name_full, + dest_name_main, + dest_name_detail, + *relevant_dest_name_detail, + dest_name_main_21, + dest_name_detail_21, + dest_name_main_19, + dest_name_detail_19, + dest_name_main_16, + dest_name_detail_16, + dest_icon, + dest_color, + dest_text_color); +} + +void Kv1Parser::parseJourneyPattern() { + auto data_owner_code = eatString("JOPA.DataOwnerCode", true, 10); + auto line_planning_number = eatString("JOPA.LinePlanningNumber", true, 10); + auto journey_pattern_code = eatString("JOPA.JourneyPatternCode", true, 10); + auto journey_pattern_type = eatString("JOPA.JourneyPatternType", true, 10); + auto direction = eatString("JOPA.Direction", true, 1); + auto description = eatString("JOPA.Description", false, 255); + if (!record_errors.empty()) return; + + if (direction != "1" && direction != "2" && direction != "A" && direction != "B") { + record_errors.push_back("JOPA.Direction should be in [1, 2, A, B]"); + return; + } + + records.journey_patterns.emplace_back( + Kv1JourneyPattern::Key( + data_owner_code, + line_planning_number, + journey_pattern_code), + journey_pattern_type, + direction[0], + description); +} + +void Kv1Parser::parseConcessionFinancerRelation() { + auto data_owner_code = eatString("CONFINREL.DataOwnerCode", true, 10); + auto con_fin_rel_code = eatString("CONFINREL.ConFinRelCode", true, 10); + auto concession_area_code = eatString("CONFINREL.ConcessionAreaCode", true, 10); + auto financer_code = eatString("CONFINREL.FinancerCode", false, 10); + if (!record_errors.empty()) return; + + records.concession_financer_relations.emplace_back( + Kv1ConcessionFinancerRelation::Key( + data_owner_code, + con_fin_rel_code), + concession_area_code, + financer_code); +} + +void Kv1Parser::parseConcessionArea() { + auto data_owner_code = eatString("CONAREA.DataOwnerCode", true, 10); + auto concession_area_code = eatString("CONAREA.ConcessionAreaCode", true, 10); + auto description = eatString("CONAREA.Description", true, 255); + if (!record_errors.empty()) return; + + records.concession_areas.emplace_back( + Kv1ConcessionArea::Key( + data_owner_code, + concession_area_code), + description); +} + +void Kv1Parser::parseFinancer() { + auto data_owner_code = eatString("FINANCER.DataOwnerCode", true, 10); + auto financer_code = eatString("FINANCER.FinancerCode", true, 10); + auto description = eatString("FINANCER.Description", true, 255); + if (!record_errors.empty()) return; + + records.financers.emplace_back( + Kv1Financer::Key( + data_owner_code, + financer_code), + description); +} + +void Kv1Parser::parseJourneyPatternTimingLink() { + auto data_owner_code = eatString ("JOPATILI.DataOwnerCode", true, 10); + auto line_planning_number = eatString ("JOPATILI.LinePlanningNumber", true, 10); + auto journey_pattern_code = eatString ("JOPATILI.JourneyPatternCode", true, 10); + auto timing_link_order = eatNumber ("JOPATILI.TimingLinkOrder", true, 3); + auto user_stop_code_begin = eatString ("JOPATILI.UserStopCodeBegin", true, 10); + auto user_stop_code_end = eatString ("JOPATILI.UserStopCodeEnd", true, 10); + auto con_fin_rel_code = eatString ("JOPATILI.ConFinRelCode", true, 10); + auto dest_code = eatString ("JOPATILI.DestCode", true, 10); + eatCell ("JOPATILI." ); + auto is_timing_stop = eatBoolean ("JOPATILI.IsTimingStop", true ); + auto display_public_line = eatString ("JOPATILI.DisplayPublicLine", false, 4); + auto product_formula_type = eatNumber ("JOPATILI.ProductFormulaType", false, 4); + auto get_in = eatBoolean ("JOPATILI.GetIn", true ); + auto get_out = eatBoolean ("JOPATILI.GetOut", true ); + auto show_flexible_trip = eatString ("JOPATILI.ShowFlexibleTrip", false, 8); + auto line_dest_icon = eatNumber ("JOPATILI.LineDestIcon", false, 4); + auto line_dest_color = eatRgbColor("JOPATILI.LineDestColor", false ); + auto line_dest_text_color = eatRgbColor("JOPATILI.LineDestTextColor", false ); + if (!record_errors.empty()) return; + + if (line_dest_icon && *line_dest_icon != static_cast(*line_dest_icon)) + record_errors.push_back("JOPATILI.LineDestIcon should be an integer"); + if (!show_flexible_trip.empty() && show_flexible_trip != "TRUE" && + show_flexible_trip != "FALSE" && show_flexible_trip != "REALTIME") + record_errors.push_back("JOPATILI.ShowFlexibleTrip should be in BISON E21 values [TRUE, FALSE, REALTIME]"); + if (!record_errors.empty()) return; + + records.journey_pattern_timing_links.emplace_back( + Kv1JourneyPatternTimingLink::Key( + data_owner_code, + line_planning_number, + journey_pattern_code, + static_cast(*timing_link_order)), + user_stop_code_begin, + user_stop_code_end, + con_fin_rel_code, + dest_code, + *is_timing_stop, + display_public_line, + product_formula_type, + *get_in, + *get_out, + show_flexible_trip, + line_dest_icon, + line_dest_color, + line_dest_text_color); +} + +void Kv1Parser::parsePoint() { + auto data_owner_code = eatString("POINT.DataOwnerCode", true, 10); + auto point_code = eatString("POINT.PointCode", true, 10); + eatCell ("POINT." ); + auto point_type = eatString("POINT.PointType", true, 10); + auto coordinate_system_type = eatString("POINT.CoordinateSystemType", true, 10); + // NOTE: We deviate from the specification here once again. The specification + // notes that LocationX_EW should contain 'at least 6 positions'. Assuming + // that this is referring to the amount of digits, we have to lower this to + // 4. Otherwise, some positions in the Netherlands and Belgium are + // unrepresentable. + auto location_x_ew = eatRdCoord("POINT.LocationX_EW", true, 4); + auto location_y_ew = eatRdCoord("POINT.LocationX_EW", true, 6); + auto location_z = eatRdCoord("POINT.LocationZ", false, 0); + auto description = eatString ("POINT.Description", false, 255); + if (!record_errors.empty()) return; + + records.points.emplace_back( + Kv1Point::Key( + std::move(data_owner_code), + std::move(point_code)), + std::move(point_type), + std::move(coordinate_system_type), + *location_x_ew, + *location_y_ew, + location_z, + std::move(description)); +} + +void Kv1Parser::parsePointOnLink() { + auto data_owner_code = eatString("POOL.DataOwnerCode", true, 10); + auto user_stop_code_begin = eatString("POOL.UserStopCodeBegin", true, 10); + auto user_stop_code_end = eatString("POOL.UserStopCodeEnd", true, 10); + eatCell ("POOL." ); + auto point_data_owner_code = eatString("POOL.PointDataOwnerCode", true, 10); + auto point_code = eatString("POOL.PointCode", true, 10); + auto distance_since_start_of_link = eatNumber("POOL.DistanceSinceStartOfLink", true, 5); + auto segment_speed = eatNumber("POOL.SegmentSpeed", false, 4); + auto local_point_speed = eatNumber("POOL.LocalPointSpeed", false, 4); + auto description = eatString("POOL.Description", false, 255); + auto transport_type = eatString("POOL.TransportType", true, 5); + if (!record_errors.empty()) return; + + records.point_on_links.emplace_back( + Kv1PointOnLink::Key( + data_owner_code, + user_stop_code_begin, + user_stop_code_end, + point_data_owner_code, + point_code, + transport_type), + *distance_since_start_of_link, + segment_speed, + local_point_speed, + std::move(description)); +} + +void Kv1Parser::parseIcon() { + auto data_owner_code = eatString("ICON.DataOwnerCode", true, 10); + auto icon_number = eatNumber("ICON.IconNumber", true, 4); + auto icon_uri = eatString("ICON.IconURI", true, 1024); + if (!record_errors.empty()) return; + + if (*icon_number != static_cast(*icon_number)) { + record_errors.push_back("ICON.IconNumber should be an integer"); + return; + } + + records.icons.emplace_back( + Kv1Icon::Key( + data_owner_code, + static_cast(*icon_number)), + icon_uri); +} + +void Kv1Parser::parseNotice() { + auto data_owner_code = eatString("NOTICE.DataOwnerCode", true, 10); + auto notice_code = eatString("NOTICE.NoticeCode", true, 20); + auto notice_content = eatString("NOTICE.NoticeContent", true, 1024); + if (!record_errors.empty()) return; + + records.notices.emplace_back( + Kv1Notice::Key( + data_owner_code, + notice_code), + notice_content); +} + +void Kv1Parser::parseNoticeAssignment() { + auto data_owner_code = eatString("NTCASSGNM.DataOwnerCode", true, 10); + auto notice_code = eatString("NTCASSGNM.NoticeCode", true, 20); + auto assigned_object = eatString("NTCASSGNM.AssignedObject", true, 8); + auto timetable_version_code = eatString("NTCASSGNM.TimetableVersionCode", false, 10); + auto organizational_unit_code = eatString("NTCASSGNM.OrganizationalUnitCode", false, 10); + auto schedule_code = eatString("NTCASSGNM.ScheduleCode", false, 10); + auto schedule_type_code = eatString("NTCASSGNM.ScheduleTypeCode", false, 10); + auto period_group_code = eatString("NTCASSGNM.PeriodGroupCode", false, 10); + auto specific_day_code = eatString("NTCASSGNM.SpecificDayCode", false, 10); + auto day_type = eatString("NTCASSGNM.DayType", false, 7); + auto line_planning_number = eatString("NTCASSGNM.LinePlanningNumber", true, 10); + auto journey_number = eatNumber("NTCASSGNM.JourneyNumber", false, 6); + auto stop_order = eatNumber("NTCASSGNM.StopOrder", false, 4); + auto journey_pattern_code = eatString("NTCASSGNM.JourneyPatternCode", false, 10); + auto timing_link_order = eatNumber("NTCASSGNM.TimingLinkOrder", false, 3); + auto user_stop_code = eatString("NTCASSGNM.UserStopCode", false, 10); + if (!record_errors.empty()) return; + + if (journey_number && *journey_number != static_cast(*journey_number)) + record_errors.push_back("NTCASSGNM.JourneyNumber should be an integer"); + if (journey_number && (*journey_number < 0 || *journey_number > 999'999)) + record_errors.push_back("NTCASSGNM.JourneyNumber should be within the range [0-999999]"); + if (stop_order && *stop_order != static_cast(*stop_order)) + record_errors.push_back("NTCASSGNM.StopOrder should be an integer"); + if (!journey_number && (assigned_object == "PUJO" || assigned_object == "PUJOPASS")) + record_errors.push_back("NTCASSGNM.JourneyNumber is required for AssignedObject PUJO/PUJOPASS"); + if (journey_pattern_code.empty() && assigned_object == "JOPATILI") + record_errors.push_back("NTCASSGNM.JourneyPatternCode is required for AssignedObject JOPATILI"); + if (!record_errors.empty()) return; + + records.notice_assignments.emplace_back( + data_owner_code, + notice_code, + assigned_object, + timetable_version_code, + organizational_unit_code, + schedule_code, + schedule_type_code, + period_group_code, + specific_day_code, + day_type, + line_planning_number, + static_cast>(journey_number), + static_cast>(stop_order), + journey_pattern_code, + timing_link_order, + user_stop_code); +} + +void Kv1Parser::parseTimeDemandGroup() { + auto data_owner_code = eatString("TIMDEMGRP.DataOwnerCode", true, 10); + auto line_planning_number = eatString("TIMDEMGRP.LinePlanningNumber", true, 10); + auto journey_pattern_code = eatString("TIMDEMGRP.JourneyPatternCode", true, 10); + auto time_demand_group_code = eatString("TIMDEMGRP.TimeDemandGroupCode", true, 10); + if (!record_errors.empty()) return; + + records.time_demand_groups.emplace_back( + Kv1TimeDemandGroup::Key( + data_owner_code, + line_planning_number, + journey_pattern_code, + time_demand_group_code)); +} + +void Kv1Parser::parseTimeDemandGroupRunTime() { + auto data_owner_code = eatString("TIMDEMRNT.DataOwnerCode", true, 10); + auto line_planning_number = eatString("TIMDEMRNT.LinePlanningNumber", true, 10); + auto journey_pattern_code = eatString("TIMDEMRNT.JourneyPatternCode", true, 10); + auto time_demand_group_code = eatString("TIMDEMRNT.TimeDemandGroupCode", true, 10); + auto timing_link_order = eatNumber("TIMDEMRNT.TimingLinkOrder", true, 3); + auto user_stop_code_begin = eatString("TIMDEMRNT.UserStopCodeBegin", true, 10); + auto user_stop_code_end = eatString("TIMDEMRNT.UserStopCodeEnd", true, 10); + auto total_drive_time = eatNumber("TIMDEMRNT.TotalDriveTime", true, 5); + auto drive_time = eatNumber("TIMDEMRNT.DriveTime", true, 5); + auto expected_delay = eatNumber("TIMDEMRNT.ExpectedDelay", false, 5); + auto layover_time = eatNumber("TIMDEMRNT.LayOverTime", false, 5); + auto stop_wait_time = eatNumber("TIMDEMRNT.StopWaitTime", true, 5); + auto minimum_stop_time = eatNumber("TIMDEMRNT.MinimumStopTime", false, 5); + if (!record_errors.empty()) return; + + if (timing_link_order && *timing_link_order != static_cast(*timing_link_order)) { + record_errors.push_back("TIMDEMRNT.TimingLinkOrder should be an integer"); + return; + } + + records.time_demand_group_run_times.emplace_back( + Kv1TimeDemandGroupRunTime::Key( + data_owner_code, + line_planning_number, + journey_pattern_code, + time_demand_group_code, + static_cast(*timing_link_order)), + user_stop_code_begin, + user_stop_code_end, + *total_drive_time, + *drive_time, + expected_delay, + layover_time, + *stop_wait_time, + minimum_stop_time); +} + +void Kv1Parser::parsePeriodGroup() { + auto data_owner_code = eatString("PEGR.DataOwnerCode", true, 10); + auto period_group_code = eatString("PEGR.PeriodGroupCode", true, 10); + auto description = eatString("PEGR.Description", false, 255); + if (!record_errors.empty()) return; + + records.period_groups.emplace_back( + Kv1PeriodGroup::Key( + data_owner_code, + period_group_code), + description); +} + +void Kv1Parser::parseSpecificDay() { + auto data_owner_code = eatString("SPECDAY.DataOwnerCode", true, 10); + auto specific_day_code = eatString("SPECDAY.SpecificDayCode", true, 10); + auto name = eatString("SPECDAY.Name", true, 50); + auto description = eatString("SPECDAY.Description", false, 255); + if (!record_errors.empty()) return; + + records.specific_days.emplace_back( + Kv1SpecificDay::Key( + data_owner_code, + specific_day_code), + name, + description); +} + +void Kv1Parser::parseTimetableVersion() { + auto data_owner_code = eatString("TIVE.DataOwnerCode", true, 10); + auto organizational_unit_code = eatString("TIVE.OrganizationalUnitCode", true, 10); + auto timetable_version_code = eatString("TIVE.TimetableVersionCode", true, 10); + auto period_group_code = eatString("TIVE.PeriodGroupCode", true, 10); + auto specific_day_code = eatString("TIVE.SpecificDayCode", true, 10); + auto valid_from_raw = eatString("TIVE.ValidFrom", true, 10); + auto timetable_version_type = eatString("TIVE.TimetableVersionType", true, 10); + auto valid_thru_raw = eatString("TIVE.ValidThru", false, 10); + auto description = eatString("TIVE.Description", false, 255); + if (!record_errors.empty()) return; + + auto valid_from = parseYyyymmdd(valid_from_raw); + if (!valid_from) + record_errors.push_back("TIVE.ValidFrom has invalid format, should be YYYY-MM-DD"); + std::optional valid_thru; + if (!valid_thru_raw.empty()) { + valid_thru = parseYyyymmdd(valid_thru_raw); + if (!valid_thru) { + record_errors.push_back("TIVE.ValidFrom has invalid format, should be YYYY-MM-DD"); + } + } + if (!description.empty()) + record_errors.push_back("TIVE.Description should be empty"); + if (!record_errors.empty()) return; + + records.timetable_versions.emplace_back( + Kv1TimetableVersion::Key( + data_owner_code, + organizational_unit_code, + timetable_version_code, + period_group_code, + specific_day_code), + *valid_from, + timetable_version_type, + valid_thru, + description); +} + +void Kv1Parser::parsePublicJourney() { + auto data_owner_code = eatString ("PUJO.DataOwnerCode", true, 10); + auto timetable_version_code = eatString ("PUJO.TimetableVersionCode", true, 10); + auto organizational_unit_code = eatString ("PUJO.OrganizationalUnitCode", true, 10); + auto period_group_code = eatString ("PUJO.PeriodGroupCode", true, 10); + auto specific_day_code = eatString ("PUJO.SpecificDayCode", true, 10); + auto day_type = eatString ("PUJO.DayType", true, 7); + auto line_planning_number = eatString ("PUJO.LinePlanningNumber", true, 10); + auto journey_number = eatNumber ("PUJO.JourneyNumber", true, 6); + auto time_demand_group_code = eatString ("PUJO.TimeDemandGroupCode", true, 10); + auto journey_pattern_code = eatString ("PUJO.JourneyPatternCode", true, 10); + auto departure_time_raw = eatString ("PUJO.DepartureTime", true, 8); + auto wheelchair_accessible = eatString ("PUJO.WheelChairAccessible", true, 13); + auto data_owner_is_operator = eatBoolean("PUJO.DataOwnerIsOperator", true ); + auto planned_monitored = eatBoolean("PUJO.PlannedMonitored", true ); + auto product_formula_type = eatNumber ("PUJO.ProductFormulaType", false, 4); + auto show_flexible_trip = eatString ("PUJO.ShowFlexibleTrip", false, 8); + if (!record_errors.empty()) return; + + auto departure_time = parseHhmmss(departure_time_raw); + if (!departure_time) + record_errors.push_back("PUJO.DepartureTime has a bad format"); + if (*journey_number < 0 || *journey_number > 999'999) + record_errors.push_back("PUJO.JourneyNumber should be within the range [0-999999]"); + if (*journey_number != static_cast(*journey_number)) + record_errors.push_back("PUJO.JourneyNumber should be an integer"); + if (product_formula_type && *product_formula_type != static_cast(*product_formula_type)) + record_errors.push_back("PUJO.ProductFormulaType should be an integer"); + if (wheelchair_accessible != "ACCESSIBLE" && wheelchair_accessible != "NOTACCESSIBLE" && wheelchair_accessible != "UNKNOWN") + record_errors.push_back("PUJO.WheelChairAccessible should be in BISON E3 values [ACCESSIBLE, NOTACCESSIBLE, UNKNOWN]"); + if (!show_flexible_trip.empty() && show_flexible_trip != "TRUE" && + show_flexible_trip != "FALSE" && show_flexible_trip != "REALTIME") + record_errors.push_back("PUJO.ShowFlexibleTrip should be in BISON E21 values [TRUE, FALSE, REALTIME]"); + if (!record_errors.empty()) return; + + records.public_journeys.emplace_back( + Kv1PublicJourney::Key( + data_owner_code, + timetable_version_code, + organizational_unit_code, + period_group_code, + specific_day_code, + day_type, + line_planning_number, + static_cast(*journey_number)), + time_demand_group_code, + journey_pattern_code, + *departure_time, + wheelchair_accessible, + *data_owner_is_operator, + *planned_monitored, + product_formula_type, + show_flexible_trip); +} + +void Kv1Parser::parsePeriodGroupValidity() { + auto data_owner_code = eatString("PEGRVAL.DataOwnerCode", true, 10); + auto organizational_unit_code = eatString("PEGRVAL.OrganizationalUnitCode", true, 10); + auto period_group_code = eatString("PEGRVAL.PeriodGroupCode", true, 10); + auto valid_from_raw = eatString("PEGRVAL.ValidFrom", true, 10); + auto valid_thru_raw = eatString("PEGRVAL.ValidThru", true, 10); + if (!record_errors.empty()) return; + + auto valid_from = parseYyyymmdd(valid_from_raw); + auto valid_thru = parseYyyymmdd(valid_thru_raw); + if (!valid_from) + record_errors.push_back("PEGRVAL.ValidFrom has invalid format, should be YYYY-MM-DD"); + if (!valid_thru) + record_errors.push_back("PEGRVAL.ValidThru has invalid format, should be YYYY-MM-DD"); + if (!record_errors.empty()) return; + + records.period_group_validities.emplace_back( + Kv1PeriodGroupValidity::Key( + data_owner_code, + organizational_unit_code, + period_group_code, + *valid_from), + *valid_thru); +} + +void Kv1Parser::parseExceptionalOperatingDay() { + auto data_owner_code = eatString("EXCOPDAY.DataOwnerCode", true, 10); + auto organizational_unit_code = eatString("EXCOPDAY.OrganizationalUnitCode", true, 10); + auto valid_date_raw = eatString("EXCOPDAY.ValidDate", true, 23); + auto day_type_as_on = eatString("EXCOPDAY.DayTypeAsOn", true, 7); + auto specific_day_code = eatString("EXCOPDAY.SpecificDayCode", true, 10); + auto period_group_code = eatString("EXCOPDAY.PeriodGroupCode", false, 10); + auto description = eatString("EXCOPDAY.Description", false, 255); + if (!record_errors.empty()) return; + + std::string_view error; + auto valid_date = parseDateTime(valid_date_raw, amsterdam, &error); + if (!valid_date) { + record_errors.push_back(std::format("EXCOPDAY.ValidDate has an bad format (value: {}): {}", valid_date_raw, error)); + return; + } + + records.exceptional_operating_days.emplace_back( + Kv1ExceptionalOperatingDay::Key( + data_owner_code, + organizational_unit_code, + *valid_date), + day_type_as_on, + specific_day_code, + period_group_code, + description); +} + +void Kv1Parser::parseScheduleVersion() { + auto data_owner_code = eatString("SCHEDVERS.DataOwnerCode", true, 10); + auto organizational_unit_code = eatString("SCHEDVERS.OrganizationalUnitCode", true, 10); + auto schedule_code = eatString("SCHEDVERS.ScheduleCode", true, 10); + auto schedule_type_code = eatString("SCHEDVERS.ScheduleTypeCode", true, 10); + auto valid_from_raw = eatString("SCHEDVERS.ValidFrom", true, 10); + auto valid_thru_raw = eatString("SCHEDVERS.ValidThru", false, 10); + auto description = eatString("SCHEDVERS.Description", false, 255); + if (!record_errors.empty()) return; + + auto valid_from = parseYyyymmdd(valid_from_raw); + if (!valid_from) + record_errors.push_back("SCHEDVERS.ValidFrom has invalid format, should be YYYY-MM-DD"); + std::optional valid_thru; + if (!valid_thru_raw.empty()) { + valid_thru = parseYyyymmdd(valid_thru_raw); + if (!valid_thru) { + record_errors.push_back("SCHEDVERS.ValidFrom has invalid format, should be YYYY-MM-DD"); + } + } + if (!description.empty()) + record_errors.push_back("SCHEDVERS.Description should be empty"); + if (!record_errors.empty()) return; + + records.schedule_versions.emplace_back( + Kv1ScheduleVersion::Key( + data_owner_code, + organizational_unit_code, + schedule_code, + schedule_type_code), + *valid_from, + valid_thru, + description); +} + +void Kv1Parser::parsePublicJourneyPassingTimes() { + auto data_owner_code = eatString ("PUJOPASS.DataOwnerCode", true, 10); + auto organizational_unit_code = eatString ("PUJOPASS.OrganizationalUnitCode", true, 10); + auto schedule_code = eatString ("PUJOPASS.ScheduleCode", true, 10); + auto schedule_type_code = eatString ("PUJOPASS.ScheduleTypeCode", true, 10); + auto line_planning_number = eatString ("PUJOPASS.LinePlanningNumber", true, 10); + auto journey_number = eatNumber ("PUJOPASS.JourneyNumber", true, 6); + auto stop_order = eatNumber ("PUJOPASS.StopOrder", true, 4); + auto journey_pattern_code = eatString ("PUJOPASS.JourneyPatternCode", true, 10); + auto user_stop_code = eatString ("PUJOPASS.UserStopCode", true, 10); + auto target_arrival_time_raw = eatString ("PUJOPASS.TargetArrivalTime", false, 8); + auto target_departure_time_raw = eatString ("PUJOPASS.TargetDepartureTime", false, 8); + auto wheelchair_accessible = eatString ("PUJOPASS.WheelChairAccessible", true, 13); + auto data_owner_is_operator = eatBoolean("PUJOPASS.DataOwnerIsOperator", true ); + auto planned_monitored = eatBoolean("PUJOPASS.PlannedMonitored", true ); + auto product_formula_type = eatNumber ("PUJOPASS.ProductFormulaType", false, 4); + auto show_flexible_trip = eatString ("PUJOPASS.ShowFlexibleTrip", false, 8); + if (!record_errors.empty()) return; + + if (*journey_number < 0 || *journey_number > 999'999) + record_errors.push_back("PUJOPASS.JourneyNumber should be within the range [0-999999]"); + if (*journey_number != static_cast(*journey_number)) + record_errors.push_back("PUJOPASS.JourneyNumber should be an integer"); + if (*stop_order != static_cast(*stop_order)) + record_errors.push_back("PUJOPASS.StopOrder should be an integer"); + if (product_formula_type && *product_formula_type != static_cast(*product_formula_type)) + record_errors.push_back("PUJOPASS.ProductFormulaType should be an integer"); + if (wheelchair_accessible != "ACCESSIBLE" && wheelchair_accessible != "NOTACCESSIBLE" && wheelchair_accessible != "UNKNOWN") + record_errors.push_back("PUJOPASS.WheelChairAccessible should be in BISON E3 values [ACCESSIBLE, NOTACCESSIBLE, UNKNOWN]"); + if (!show_flexible_trip.empty() && show_flexible_trip != "TRUE" && + show_flexible_trip != "FALSE" && show_flexible_trip != "REALTIME") + record_errors.push_back("PUJOPASS.ShowFlexibleTrip should be in BISON E21 values [TRUE, FALSE, REALTIME]"); + std::optional> target_arrival_time; + if (!target_arrival_time_raw.empty()) { + target_arrival_time = parseHhmmss(target_arrival_time_raw); + if (!target_arrival_time) { + record_errors.push_back("PUJOPASS.TargetArrivalTime has invalid format, should be HH:MM:SS"); + } + } + std::optional> target_departure_time; + if (!target_departure_time_raw.empty()) { + target_departure_time = parseHhmmss(target_departure_time_raw); + if (!target_departure_time) { + record_errors.push_back("PUJOPASS.TargetDepartureTime has invalid format, should be HH:MM:SS"); + } + } + if (!record_errors.empty()) return; + + records.public_journey_passing_times.emplace_back( + Kv1PublicJourneyPassingTimes::Key( + data_owner_code, + organizational_unit_code, + schedule_code, + schedule_type_code, + line_planning_number, + static_cast(*journey_number), + static_cast(*stop_order)), + journey_pattern_code, + user_stop_code, + target_arrival_time, + target_departure_time, + wheelchair_accessible, + *data_owner_is_operator, + *planned_monitored, + product_formula_type, + show_flexible_trip); +} + +void Kv1Parser::parseOperatingDay() { + auto data_owner_code = eatString("OPERDAY.DataOwnerCode", true, 10); + auto organizational_unit_code = eatString("OPERDAY.OrganizationalUnitCode", true, 10); + auto schedule_code = eatString("OPERDAY.ScheduleCode", true, 10); + auto schedule_type_code = eatString("OPERDAY.ScheduleTypeCode", true, 10); + auto valid_date_raw = eatString("OPERDAY.ValidDate", true, 10); + auto description = eatString("OPERDAY.Description", false, 255); + if (!record_errors.empty()) return; + + auto valid_date = parseYyyymmdd(valid_date_raw); + if (!valid_date) + record_errors.push_back("OPERDAY.ValidDate has invalid format, should be YYYY-MM-DD"); + if (!record_errors.empty()) return; + + records.operating_days.emplace_back( + Kv1OperatingDay::Key( + data_owner_code, + organizational_unit_code, + schedule_code, + schedule_type_code, + *valid_date), + description); +} + +const std::unordered_map Kv1Parser::type_parsers{ + { "ORUN", &Kv1Parser::parseOrganizationalUnit }, + { "ORUNORUN", &Kv1Parser::parseHigherOrganizationalUnit }, + { "USRSTOP", &Kv1Parser::parseUserStopPoint }, + { "USRSTAR", &Kv1Parser::parseUserStopArea }, + { "TILI", &Kv1Parser::parseTimingLink }, + { "LINK", &Kv1Parser::parseLink }, + { "LINE", &Kv1Parser::parseLine }, + { "DEST", &Kv1Parser::parseDestination }, + { "JOPA", &Kv1Parser::parseJourneyPattern }, + { "CONFINREL", &Kv1Parser::parseConcessionFinancerRelation }, + { "CONAREA", &Kv1Parser::parseConcessionArea }, + { "FINANCER", &Kv1Parser::parseFinancer }, + { "JOPATILI", &Kv1Parser::parseJourneyPatternTimingLink }, + { "POINT", &Kv1Parser::parsePoint }, + { "POOL", &Kv1Parser::parsePointOnLink }, + { "ICON", &Kv1Parser::parseIcon }, + { "NOTICE", &Kv1Parser::parseNotice }, + { "NTCASSGNM", &Kv1Parser::parseNoticeAssignment }, + { "TIMDEMGRP", &Kv1Parser::parseTimeDemandGroup }, + { "TIMDEMRNT", &Kv1Parser::parseTimeDemandGroupRunTime }, + { "PEGR", &Kv1Parser::parsePeriodGroup }, + { "SPECDAY", &Kv1Parser::parseSpecificDay }, + { "TIVE", &Kv1Parser::parseTimetableVersion }, + { "PUJO", &Kv1Parser::parsePublicJourney }, + { "PEGRVAL", &Kv1Parser::parsePeriodGroupValidity }, + { "EXCOPDAY", &Kv1Parser::parseExceptionalOperatingDay }, + { "SCHEDVERS", &Kv1Parser::parseScheduleVersion }, + { "PUJOPASS", &Kv1Parser::parsePublicJourneyPassingTimes }, + { "OPERDAY", &Kv1Parser::parseOperatingDay }, +}; diff --git a/lib/libtmi8/src/kv1_types.cpp b/lib/libtmi8/src/kv1_types.cpp new file mode 100644 index 0000000..49e306e --- /dev/null +++ b/lib/libtmi8/src/kv1_types.cpp @@ -0,0 +1,773 @@ +// vim:set sw=2 ts=2 sts et: + +#include + +#include + +size_t Kv1Records::size() const { + return organizational_units.size() + + higher_organizational_units.size() + + user_stop_points.size() + + user_stop_areas.size() + + timing_links.size() + + links.size() + + lines.size() + + destinations.size() + + journey_patterns.size() + + concession_financer_relations.size() + + concession_areas.size() + + financers.size() + + journey_pattern_timing_links.size() + + points.size() + + point_on_links.size() + + icons.size() + + notices.size() + + notice_assignments.size() + + time_demand_groups.size() + + time_demand_group_run_times.size() + + period_groups.size() + + specific_days.size() + + timetable_versions.size() + + public_journeys.size() + + period_group_validities.size() + + exceptional_operating_days.size() + + schedule_versions.size() + + public_journey_passing_times.size() + + operating_days.size(); +} + +Kv1OrganizationalUnit::Key::Key( + std::string data_owner_code, + std::string organizational_unit_code) + : data_owner_code(std::move(data_owner_code)), + organizational_unit_code(std::move(organizational_unit_code)) +{} + +Kv1HigherOrganizationalUnit::Key::Key( + std::string data_owner_code, + std::string organizational_unit_code_parent, + std::string organizational_unit_code_child, + std::chrono::year_month_day valid_from) + : data_owner_code(std::move(data_owner_code)), + organizational_unit_code_parent(std::move(organizational_unit_code_parent)), + organizational_unit_code_child(std::move(organizational_unit_code_child)), + valid_from(valid_from) +{} + +Kv1UserStopPoint::Key::Key( + std::string data_owner_code, + std::string user_stop_code) + : data_owner_code(std::move(data_owner_code)), + user_stop_code(std::move(user_stop_code)) +{} + +Kv1UserStopArea::Key::Key( + std::string data_owner_code, + std::string user_stop_area_code) + : data_owner_code(std::move(data_owner_code)), + user_stop_area_code(std::move(user_stop_area_code)) +{} + +Kv1TimingLink::Key::Key( + std::string data_owner_code, + std::string user_stop_code_begin, + std::string user_stop_code_end) + : data_owner_code(std::move(data_owner_code)), + user_stop_code_begin(std::move(user_stop_code_begin)), + user_stop_code_end(std::move(user_stop_code_end)) +{} + +Kv1Link::Key::Key(std::string data_owner_code, + std::string user_stop_code_begin, + std::string user_stop_code_end, + std::string transport_type) + : data_owner_code(std::move(data_owner_code)), + user_stop_code_begin(std::move(user_stop_code_begin)), + user_stop_code_end(std::move(user_stop_code_end)), + transport_type(std::move(transport_type)) +{} + +Kv1Line::Key::Key(std::string data_owner_code, + std::string line_planning_number) + : data_owner_code(std::move(data_owner_code)), + line_planning_number(std::move(line_planning_number)) +{} + +Kv1Destination::Key::Key(std::string data_owner_code, + std::string dest_code) + : data_owner_code(std::move(data_owner_code)), + dest_code(std::move(dest_code)) +{} + +Kv1JourneyPattern::Key::Key(std::string data_owner_code, + std::string line_planning_number, + std::string journey_pattern_code) + : data_owner_code(std::move(data_owner_code)), + line_planning_number(std::move(line_planning_number)), + journey_pattern_code(std::move(journey_pattern_code)) +{} + +Kv1ConcessionFinancerRelation::Key::Key(std::string data_owner_code, + std::string con_fin_rel_code) + : data_owner_code(std::move(data_owner_code)), + con_fin_rel_code(std::move(con_fin_rel_code)) +{} + +Kv1ConcessionArea::Key::Key(std::string data_owner_code, + std::string concession_area_code) + : data_owner_code(std::move(data_owner_code)), + concession_area_code(std::move(concession_area_code)) +{} + +Kv1Financer::Key::Key(std::string data_owner_code, + std::string financer_code) + : data_owner_code(std::move(data_owner_code)), + financer_code(std::move(financer_code)) +{} + +Kv1JourneyPatternTimingLink::Key::Key(std::string data_owner_code, + std::string line_planning_number, + std::string journey_pattern_code, + short timing_link_order) + : data_owner_code(std::move(data_owner_code)), + line_planning_number(std::move(line_planning_number)), + journey_pattern_code(journey_pattern_code), + timing_link_order(timing_link_order) +{} + +Kv1Point::Key::Key(std::string data_owner_code, + std::string point_code) + : data_owner_code(std::move(data_owner_code)), + point_code(std::move(point_code)) +{} + +Kv1PointOnLink::Key::Key(std::string data_owner_code, + std::string user_stop_code_begin, + std::string user_stop_code_end, + std::string point_data_owner_code, + std::string point_code, + std::string transport_type) + : data_owner_code(std::move(data_owner_code)), + user_stop_code_begin(std::move(user_stop_code_begin)), + user_stop_code_end(std::move(user_stop_code_end)), + point_data_owner_code(std::move(point_data_owner_code)), + point_code(std::move(point_code)), + transport_type(std::move(transport_type)) +{} + +Kv1Icon::Key::Key(std::string data_owner_code, + short icon_number) + : data_owner_code(std::move(data_owner_code)), + icon_number(icon_number) +{} + +Kv1Notice::Key::Key(std::string data_owner_code, + std::string notice_code) + : data_owner_code(std::move(data_owner_code)), + notice_code(std::move(notice_code)) +{} + +Kv1TimeDemandGroup::Key::Key(std::string data_owner_code, + std::string line_planning_number, + std::string journey_pattern_code, + std::string time_demand_group_code) + : data_owner_code(std::move(data_owner_code)), + line_planning_number(std::move(line_planning_number)), + journey_pattern_code(std::move(journey_pattern_code)), + time_demand_group_code(std::move(time_demand_group_code)) +{} + +Kv1TimeDemandGroupRunTime::Key::Key(std::string data_owner_code, + std::string line_planning_number, + std::string journey_pattern_code, + std::string time_demand_group_code, + short timing_link_order) + : data_owner_code(std::move(data_owner_code)), + line_planning_number(std::move(line_planning_number)), + journey_pattern_code(std::move(journey_pattern_code)), + time_demand_group_code(std::move(time_demand_group_code)), + timing_link_order(std::move(timing_link_order)) +{} + +Kv1PeriodGroup::Key::Key(std::string data_owner_code, + std::string period_group_code) + : data_owner_code(std::move(data_owner_code)), + period_group_code(std::move(period_group_code)) +{} + +Kv1SpecificDay::Key::Key(std::string data_owner_code, + std::string specific_day_code) + : data_owner_code(std::move(data_owner_code)), + specific_day_code(std::move(specific_day_code)) +{} + +Kv1TimetableVersion::Key::Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string timetable_version_code, + std::string period_group_code, + std::string specific_day_code) + : data_owner_code(std::move(data_owner_code)), + organizational_unit_code(std::move(organizational_unit_code)), + timetable_version_code(std::move(timetable_version_code)), + period_group_code(std::move(period_group_code)), + specific_day_code(std::move(specific_day_code)) +{} + +Kv1PublicJourney::Key::Key(std::string data_owner_code, + std::string timetable_version_code, + std::string organizational_unit_code, + std::string period_group_code, + std::string specific_day_code, + std::string day_type, + std::string line_planning_number, + int journey_number) + : data_owner_code(std::move(data_owner_code)), + timetable_version_code(std::move(timetable_version_code)), + organizational_unit_code(std::move(organizational_unit_code)), + period_group_code(std::move(period_group_code)), + specific_day_code(std::move(specific_day_code)), + day_type(std::move(day_type)), + line_planning_number(std::move(line_planning_number)), + journey_number(journey_number) +{} + +Kv1PeriodGroupValidity::Key::Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string period_group_code, + std::chrono::year_month_day valid_from) + : data_owner_code(std::move(data_owner_code)), + organizational_unit_code(std::move(organizational_unit_code)), + period_group_code(std::move(period_group_code)), + valid_from(valid_from) +{} + +Kv1ExceptionalOperatingDay::Key::Key(std::string data_owner_code, + std::string organizational_unit_code, + std::chrono::sys_seconds valid_date) + : data_owner_code(std::move(data_owner_code)), + organizational_unit_code(std::move(organizational_unit_code)), + valid_date(valid_date) +{} + +Kv1ScheduleVersion::Key::Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string schedule_code, + std::string schedule_type_code) + : data_owner_code(std::move(data_owner_code)), + organizational_unit_code(std::move(organizational_unit_code)), + schedule_code(std::move(schedule_code)), + schedule_type_code(std::move(schedule_type_code)) +{} + +Kv1PublicJourneyPassingTimes::Key::Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string schedule_code, + std::string schedule_type_code, + std::string line_planning_number, + int journey_number, + short stop_order) + : data_owner_code(std::move(data_owner_code)), + organizational_unit_code(std::move(organizational_unit_code)), + schedule_code(std::move(schedule_code)), + schedule_type_code(std::move(schedule_type_code)), + line_planning_number(std::move(line_planning_number)), + journey_number(journey_number), + stop_order(stop_order) +{} + +Kv1OperatingDay::Key::Key(std::string data_owner_code, + std::string organizational_unit_code, + std::string schedule_code, + std::string schedule_type_code, + std::chrono::year_month_day valid_date) + : data_owner_code(std::move(data_owner_code)), + organizational_unit_code(std::move(organizational_unit_code)), + schedule_code(std::move(schedule_code)), + schedule_type_code(std::move(schedule_type_code)), + valid_date(valid_date) +{} + +bool operator==(const Kv1OrganizationalUnit::Key &a, const Kv1OrganizationalUnit::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.organizational_unit_code == b.organizational_unit_code; +} + +bool operator==(const Kv1HigherOrganizationalUnit::Key &a, const Kv1HigherOrganizationalUnit::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.organizational_unit_code_parent == b.organizational_unit_code_parent + && a.organizational_unit_code_child == b.organizational_unit_code_child; +} + +bool operator==(const Kv1UserStopPoint::Key &a, const Kv1UserStopPoint::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.user_stop_code == b.user_stop_code; +} + +bool operator==(const Kv1UserStopArea::Key &a, const Kv1UserStopArea::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.user_stop_area_code == b.user_stop_area_code; +} + +bool operator==(const Kv1TimingLink::Key &a, const Kv1TimingLink::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.user_stop_code_begin == b.user_stop_code_begin + && a.user_stop_code_end == b.user_stop_code_end; +} + +bool operator==(const Kv1Link::Key &a, const Kv1Link::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.user_stop_code_begin == b.user_stop_code_begin + && a.user_stop_code_end == b.user_stop_code_end + && a.transport_type == b.transport_type; +} + +bool operator==(const Kv1Line::Key &a, const Kv1Line::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.line_planning_number == b.line_planning_number; +} + +bool operator==(const Kv1Destination::Key &a, const Kv1Destination::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.dest_code == b.dest_code; +} + +bool operator==(const Kv1JourneyPattern::Key &a, const Kv1JourneyPattern::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.line_planning_number == b.line_planning_number + && a.journey_pattern_code == b.journey_pattern_code; +} + +bool operator==(const Kv1ConcessionFinancerRelation::Key &a, const Kv1ConcessionFinancerRelation::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.con_fin_rel_code == b.con_fin_rel_code; +} + +bool operator==(const Kv1ConcessionArea::Key &a, const Kv1ConcessionArea::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.concession_area_code == b.concession_area_code; +} + +bool operator==(const Kv1Financer::Key &a, const Kv1Financer::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.financer_code == b.financer_code; +} + +bool operator==(const Kv1JourneyPatternTimingLink::Key &a, const Kv1JourneyPatternTimingLink::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.line_planning_number == b.line_planning_number + && a.journey_pattern_code == b.journey_pattern_code + && a.timing_link_order == b.timing_link_order; +} + +bool operator==(const Kv1Point::Key &a, const Kv1Point::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.point_code == b.point_code; +} + +bool operator==(const Kv1PointOnLink::Key &a, const Kv1PointOnLink::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.user_stop_code_begin == b.user_stop_code_begin + && a.user_stop_code_end == b.user_stop_code_end + && a.point_data_owner_code == b.point_data_owner_code + && a.point_code == b.point_code + && a.transport_type == b.transport_type; +} + +bool operator==(const Kv1Icon::Key &a, const Kv1Icon::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.icon_number == b.icon_number; +} + +bool operator==(const Kv1Notice::Key &a, const Kv1Notice::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.notice_code == b.notice_code; +} + +bool operator==(const Kv1TimeDemandGroup::Key &a, const Kv1TimeDemandGroup::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.line_planning_number == b.line_planning_number + && a.journey_pattern_code == b.journey_pattern_code + && a.time_demand_group_code == b.time_demand_group_code; +} + +bool operator==(const Kv1TimeDemandGroupRunTime::Key &a, const Kv1TimeDemandGroupRunTime::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.line_planning_number == b.line_planning_number + && a.journey_pattern_code == b.journey_pattern_code + && a.time_demand_group_code == b.time_demand_group_code + && a.timing_link_order == b.timing_link_order; +} + +bool operator==(const Kv1PeriodGroup::Key &a, const Kv1PeriodGroup::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.period_group_code == b.period_group_code; +} + +bool operator==(const Kv1SpecificDay::Key &a, const Kv1SpecificDay::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.specific_day_code == b.specific_day_code; +} + +bool operator==(const Kv1TimetableVersion::Key &a, const Kv1TimetableVersion::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.organizational_unit_code == b.organizational_unit_code + && a.timetable_version_code == b.timetable_version_code + && a.period_group_code == b.period_group_code + && a.specific_day_code == b.specific_day_code; +} + +bool operator==(const Kv1PublicJourney::Key &a, const Kv1PublicJourney::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.timetable_version_code == b.timetable_version_code + && a.organizational_unit_code == b.organizational_unit_code + && a.period_group_code == b.period_group_code + && a.specific_day_code == b.specific_day_code + && a.day_type == b.day_type + && a.line_planning_number == b.line_planning_number + && a.journey_number == b.journey_number; +} + +bool operator==(const Kv1PeriodGroupValidity::Key &a, const Kv1PeriodGroupValidity::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.organizational_unit_code == b.organizational_unit_code + && a.period_group_code == b.period_group_code + && a.valid_from == b.valid_from; +} + +bool operator==(const Kv1ExceptionalOperatingDay::Key &a, const Kv1ExceptionalOperatingDay::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.organizational_unit_code == b.organizational_unit_code + && a.valid_date == b.valid_date; +} + +bool operator==(const Kv1ScheduleVersion::Key &a, const Kv1ScheduleVersion::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.organizational_unit_code == b.organizational_unit_code + && a.schedule_code == b.schedule_code + && a.schedule_type_code == b.schedule_type_code; +} + +bool operator==(const Kv1PublicJourneyPassingTimes::Key &a, const Kv1PublicJourneyPassingTimes::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.organizational_unit_code == b.organizational_unit_code + && a.schedule_code == b.schedule_code + && a.schedule_type_code == b.schedule_type_code + && a.line_planning_number == b.line_planning_number + && a.journey_number == b.journey_number + && a.stop_order == b.stop_order; +} + +bool operator==(const Kv1OperatingDay::Key &a, const Kv1OperatingDay::Key &b) { + return a.data_owner_code == b.data_owner_code + && a.organizational_unit_code == b.organizational_unit_code + && a.schedule_code == b.schedule_code + && a.schedule_type_code == b.schedule_type_code + && a.valid_date == b.valid_date; +} + +namespace std::chrono { + static size_t hash_value(const year_month_day &ymd) { + size_t seed = 0; + + boost::hash_combine(seed, int(ymd.year())); + boost::hash_combine(seed, unsigned(ymd.month())); + boost::hash_combine(seed, unsigned(ymd.day())); + + return seed; + } + + static size_t hash_value(const sys_seconds &s) { + return boost::hash()(s.time_since_epoch().count()); + } +} + +size_t hash_value(const Kv1OrganizationalUnit::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.organizational_unit_code); + + return seed; +} + +size_t hash_value(const Kv1HigherOrganizationalUnit::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.organizational_unit_code_parent); + boost::hash_combine(seed, k.organizational_unit_code_child); + boost::hash_combine(seed, k.valid_from); + + return seed; +} + +size_t hash_value(const Kv1UserStopPoint::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.user_stop_code); + + return seed; +} + +size_t hash_value(const Kv1UserStopArea::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.user_stop_area_code); + + return seed; +} + +size_t hash_value(const Kv1TimingLink::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.user_stop_code_begin); + boost::hash_combine(seed, k.user_stop_code_end); + + return seed; +} + +size_t hash_value(const Kv1Link::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.user_stop_code_begin); + boost::hash_combine(seed, k.user_stop_code_end); + boost::hash_combine(seed, k.transport_type); + + return seed; +} + +size_t hash_value(const Kv1Line::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.line_planning_number); + + return seed; +} + +size_t hash_value(const Kv1Destination::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.dest_code); + + return seed; +} + +size_t hash_value(const Kv1JourneyPattern::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.line_planning_number); + boost::hash_combine(seed, k.journey_pattern_code); + + return seed; +} + +size_t hash_value(const Kv1ConcessionFinancerRelation::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.con_fin_rel_code); + + return seed; +} + +size_t hash_value(const Kv1ConcessionArea::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.concession_area_code); + + return seed; +} + +size_t hash_value(const Kv1Financer::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.financer_code); + + return seed; +} + +size_t hash_value(const Kv1JourneyPatternTimingLink::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.line_planning_number); + boost::hash_combine(seed, k.journey_pattern_code); + boost::hash_combine(seed, k.timing_link_order); + + return seed; +} + +size_t hash_value(const Kv1Point::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.point_code); + + return seed; +} + +size_t hash_value(const Kv1PointOnLink::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.user_stop_code_begin); + boost::hash_combine(seed, k.user_stop_code_end); + boost::hash_combine(seed, k.point_data_owner_code); + boost::hash_combine(seed, k.point_code); + boost::hash_combine(seed, k.transport_type); + + return seed; +} + +size_t hash_value(const Kv1Icon::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.icon_number); + + return seed; +} + +size_t hash_value(const Kv1Notice::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.notice_code); + + return seed; +} + +size_t hash_value(const Kv1TimeDemandGroup::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.line_planning_number); + boost::hash_combine(seed, k.journey_pattern_code); + boost::hash_combine(seed, k.time_demand_group_code); + + return seed; +} + +size_t hash_value(const Kv1TimeDemandGroupRunTime::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.line_planning_number); + boost::hash_combine(seed, k.journey_pattern_code); + boost::hash_combine(seed, k.time_demand_group_code); + boost::hash_combine(seed, k.timing_link_order); + + return seed; +} + +size_t hash_value(const Kv1PeriodGroup::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.period_group_code); + + return seed; +} + +size_t hash_value(const Kv1SpecificDay::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.specific_day_code); + + return seed; +} + +size_t hash_value(const Kv1TimetableVersion::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.organizational_unit_code); + boost::hash_combine(seed, k.timetable_version_code); + boost::hash_combine(seed, k.period_group_code); + boost::hash_combine(seed, k.specific_day_code); + + return seed; +} + +size_t hash_value(const Kv1PublicJourney::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.timetable_version_code); + boost::hash_combine(seed, k.organizational_unit_code); + boost::hash_combine(seed, k.period_group_code); + boost::hash_combine(seed, k.specific_day_code); + boost::hash_combine(seed, k.day_type); + boost::hash_combine(seed, k.line_planning_number); + boost::hash_combine(seed, k.journey_number); + + return seed; +} + +size_t hash_value(const Kv1PeriodGroupValidity::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.organizational_unit_code); + boost::hash_combine(seed, k.period_group_code); + boost::hash_combine(seed, k.valid_from); + + return seed; +} + +size_t hash_value(const Kv1ExceptionalOperatingDay::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.organizational_unit_code); + boost::hash_combine(seed, k.valid_date); + + return seed; +} + +size_t hash_value(const Kv1ScheduleVersion::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.organizational_unit_code); + boost::hash_combine(seed, k.schedule_code); + boost::hash_combine(seed, k.schedule_type_code); + + return seed; +} + +size_t hash_value(const Kv1PublicJourneyPassingTimes::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.organizational_unit_code); + boost::hash_combine(seed, k.schedule_code); + boost::hash_combine(seed, k.schedule_type_code); + boost::hash_combine(seed, k.line_planning_number); + boost::hash_combine(seed, k.journey_number); + boost::hash_combine(seed, k.stop_order); + + return seed; +} + +size_t hash_value(const Kv1OperatingDay::Key &k) { + size_t seed = 0; + + boost::hash_combine(seed, k.data_owner_code); + boost::hash_combine(seed, k.organizational_unit_code); + boost::hash_combine(seed, k.schedule_code); + boost::hash_combine(seed, k.schedule_type_code); + boost::hash_combine(seed, k.valid_date); + + return seed; +} diff --git a/lib/libtmi8/src/kv6_parquet.cpp b/lib/libtmi8/src/kv6_parquet.cpp new file mode 100644 index 0000000..ca70b7f --- /dev/null +++ b/lib/libtmi8/src/kv6_parquet.cpp @@ -0,0 +1,102 @@ +// vim:set sw=2 ts=2 sts et: + +#include + +ParquetBuilder::ParquetBuilder() { + std::shared_ptr field_type, field_data_owner_code, field_line_planning_number, field_operating_day, + field_journey_number, field_reinforcement_number, field_timestamp, field_source, + field_punctuality, field_user_stop_code, field_passage_sequence_number, + field_vehicle_number, field_block_code, field_wheelchair_accessible, + field_number_of_coaches, field_rd_y, field_rd_x, field_distance_since_last_user_stop; + field_type = arrow::field("type", arrow::utf8()); + field_data_owner_code = arrow::field("data_owner_code", arrow::utf8()); + field_line_planning_number = arrow::field("line_planning_number", arrow::utf8()); + field_operating_day = arrow::field("operating_day", arrow::date32()); + field_journey_number = arrow::field("journey_number", arrow::uint32()); + field_reinforcement_number = arrow::field("reinforcement_number", arrow::uint8()); + field_timestamp = arrow::field("timestamp", arrow::timestamp(arrow::TimeUnit::SECOND)); + field_source = arrow::field("source", arrow::utf8()); + field_punctuality = arrow::field("punctuality", arrow::int16()); + field_user_stop_code = arrow::field("user_stop_code", arrow::utf8()); + field_passage_sequence_number = arrow::field("passage_sequence_number", arrow::uint16()); + field_vehicle_number = arrow::field("vehicle_number", arrow::uint32()); + field_block_code = arrow::field("block_code", arrow::uint32()); + field_wheelchair_accessible = arrow::field("wheelchair_accessible", arrow::utf8()); + field_number_of_coaches = arrow::field("number_of_coaches", arrow::uint8()); + field_rd_y = arrow::field("rd_y", arrow::int32()); + field_rd_x = arrow::field("rd_x", arrow::int32()); + field_distance_since_last_user_stop = arrow::field("distance_since_last_user_stop", arrow::uint32()); + + schema = arrow::schema({ field_type, field_data_owner_code, field_line_planning_number, + field_operating_day, field_journey_number, + field_reinforcement_number, field_timestamp, field_source, + field_punctuality, field_user_stop_code, + field_passage_sequence_number, field_vehicle_number, + field_block_code, field_wheelchair_accessible, + field_number_of_coaches, field_rd_y, field_rd_x, + field_distance_since_last_user_stop }); +} + +arrow::Result> ParquetBuilder::getTable() { + ARROW_ASSIGN_OR_RAISE(std::shared_ptr types, types.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr data_owner_codes, data_owner_codes.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr line_planning_numbers, line_planning_numbers.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr operating_days, operating_days.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr journey_numbers, journey_numbers.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr reinforcement_numbers, reinforcement_numbers.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr timestamps, timestamps.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr sources, sources.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr punctualities, punctualities.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr user_stop_codes, user_stop_codes.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr passage_sequence_numbers, passage_sequence_numbers.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr vehicle_numbers, vehicle_numbers.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr block_codes, block_codes.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr wheelchair_accessibles, wheelchair_accessibles.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr number_of_coaches, number_of_coaches.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr rd_ys, rd_ys.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr rd_xs, rd_xs.Finish()); + ARROW_ASSIGN_OR_RAISE(std::shared_ptr distance_since_last_user_stops, distance_since_last_user_stops.Finish()); + + std::vector> columns = { types, data_owner_codes, line_planning_numbers, operating_days, + journey_numbers, reinforcement_numbers, timestamps, sources, + punctualities, user_stop_codes, passage_sequence_numbers, + vehicle_numbers, block_codes, wheelchair_accessibles, + number_of_coaches, rd_ys, rd_xs, + distance_since_last_user_stops }; + return arrow::Result(arrow::Table::Make(schema, columns)); +} + +arrow::Status writeArrowRecordsAsParquetFile(arrow::RecordBatchReader &rbr, std::filesystem::path filename) { + std::shared_ptr props = parquet::WriterProperties::Builder() + .compression(arrow::Compression::ZSTD) + ->created_by("oeuf-libtmi8") + ->version(parquet::ParquetVersion::PARQUET_2_6) + ->data_page_version(parquet::ParquetDataPageVersion::V2) + ->max_row_group_length(MAX_PARQUET_CHUNK) + ->build(); + + std::shared_ptr arrow_props = parquet::ArrowWriterProperties::Builder() + .store_schema()->build(); + + std::shared_ptr out_file; + std::string filename_str = filename; + ARROW_ASSIGN_OR_RAISE(out_file, arrow::io::FileOutputStream::Open(filename_str + ".part")); + + ARROW_ASSIGN_OR_RAISE(auto writer, + parquet::arrow::FileWriter::Open(*rbr.schema(), arrow::default_memory_pool(), out_file, props, arrow_props)); + for (const auto &batchr : rbr) { + ARROW_ASSIGN_OR_RAISE(auto batch, batchr); + ARROW_RETURN_NOT_OK(writer->WriteRecordBatch(*batch)); + } + ARROW_RETURN_NOT_OK(writer->Close()); + ARROW_RETURN_NOT_OK(out_file->Close()); + + std::filesystem::rename(filename_str + ".part", filename); + + return arrow::Status::OK(); +} + +arrow::Status writeArrowTableAsParquetFile(const arrow::Table &table, std::filesystem::path filename) { + auto tbr = arrow::TableBatchReader(table); + return writeArrowRecordsAsParquetFile(tbr, filename); +} -- cgit v1.2.3