Unverified Commit d58f23c3 authored by networkException's avatar networkException
Browse files

Initial MOTIS backend support 

This patch adds support for checkins using MOTIS backends
using the Travel::Status::MOTIS module.

With this travelynx supports the two services currently
exposed by the module, RNV for local transit in Mannheim,
Germany and surrounding cities and transitous for worldwide
crowdsourced tranit feeds.

This implementation supports realtime predictions,
cancellations and polylines as well as custom route colors
if available.

As MOTIS doesn't expose names of indivial trips currently,
displaying transports is mostly limited to route names.

MOTIS uses strings for stop ids, based on the used GTFS
source feeds. As travelynx's data model currently assumes
interger station ids, this patch adds a mapping table
to the database.

This patch assumes support for MOTIS in db-fakedisplay.

Note that while träwelling has migrated to tranitous fully
sync remains unsupported for now.

See https://github.com/Traewelling/traewelling/issues/3345
parent 51b0080b

lib/Travelynx.pm

+168 −4
Original line number Diff line number Diff line 
package Travelynx;



# Copyright (C) 2020-2023 Birte Kristina Friesel

# Copyright (C) 2025 networkException <git@nwex.de>

#

# SPDX-License-Identifier: AGPL-3.0-or-later
@@ -24,6 +25,7 @@ use Travelynx::Helper::DBDB; 
use Travelynx::Helper::DBRIS;

use Travelynx::Helper::HAFAS;

use Travelynx::Helper::IRIS;

use Travelynx::Helper::MOTIS;

use Travelynx::Helper::Sendmail;

use Travelynx::Helper::Traewelling;

use Travelynx::Model::InTransit;
@@ -259,6 +261,18 @@ sub startup { 
		}

	);



	$self->helper(

		motis => sub {

			my ($self) = @_;

			state $motis = Travelynx::Helper::MOTIS->new(

				log        => $self->app->log,

				cache      => $self->app->cache_iris_rt,

				user_agent => $self->ua,

				version    => $self->app->config->{version},

			);

		}

	);



	$self->helper(

		traewelling => sub {

			my ($self) = @_;
@@ -475,6 +489,9 @@ sub startup { 
				return Mojo::Promise->reject('You are already checked in');

			}



			if ( $opt{motis} ) {

				return $self->_checkin_motis_p(%opt);

			}

			if ( $opt{dbris} ) {

				return $self->_checkin_dbris_p(%opt);

			}
@@ -556,6 +573,147 @@ sub startup { 
		}

	);



	$self->helper(

		'_checkin_motis_p' => sub {

			my ( $self, %opt ) = @_;



			my $station  = $opt{station};

			my $train_id = $opt{train_id};

			my $ts       = $opt{ts};

			my $uid      = $opt{uid} // $self->current_user->{id};

			my $db       = $opt{db}  // $self->pg->db;

			my $hafas;



			my $promise = Mojo::Promise->new;



			$self->motis->get_trip_p(

				service => $opt{motis},

				trip_id => $train_id,

			)->then(

				sub {

					my ($trip) = @_;

					my $found_stopover;



					for my $stopover ( $trip->stopovers ) {

						if ( $stopover->stop->id eq $station ) {

							$found_stopover = $stopover;



							# Lines may serve the same stop several times.

							# Keep looking until the scheduled departure

							# matches the one passed while checking in.

							if ( $ts and $stopover->scheduled_departure->epoch == $ts ) {

								last;

							}

						}

					}



					if ( not $found_stopover ) {

						$promise->reject("Did not find stopover at '$station' within trip '$train_id'");

						return;

					}



					for my $stopover ( $trip->stopovers ) {

						$self->stations->add_or_update(

							stop  => $stopover->stop,

							db    => $db,

							motis => $opt{motis},

						);

					}



					$self->stations->add_or_update(

						stop  => $found_stopover->stop,

						db    => $db,

						motis => $opt{motis},

					);



					eval {

						$self->in_transit->add(

							uid        => $uid,

							db         => $db,

							journey    => $trip,

							stopover   => $found_stopover,

							data       => { trip_id => $train_id },

							backend_id => $self->stations->get_backend_id(

								motis => $opt{motis}

							),

						);

					};



					if ($@) {

						$self->app->log->error("Checkin($uid): INSERT failed: $@");

						$promise->reject( 'INSERT failed: ' . $@ );

						return;

					}



					my $polyline;

					if ( $trip->polyline ) {

						my @station_list;

						my @coordinate_list;

						for my $coordinate ( $trip->polyline ) {

							if ( $coordinate->{stop} ) {

								if ( not defined $coordinate->{stop}->{eva} ) {

									die()

								}



								push(

									@coordinate_list,

									[

										$coordinate->{lon}, $coordinate->{lat},

										$coordinate->{stop}->{eva}

									]

								);



								push( @station_list, $coordinate->{stop}->name );

							}

							else {

								push( @coordinate_list, [ $coordinate->{lon}, $coordinate->{lat} ] );

							}

						}



						# equal length → polyline only consists of straight

						# lines between stops. that's not helpful.

						if ( @station_list == @coordinate_list ) {

							$self->log->debug( 'Ignoring polyline for '

								  . $trip->route_name

								  . ' as it only consists of straight lines between stops.'

							);

						}

						else {

							$polyline = {

								from_eva => ( $trip->stopovers )[0]->stop->{eva},

								to_eva   => ( $trip->stopovers )[-1]->stop->{eva},

								coords   => \@coordinate_list,

							};

						}

					}



					if ($polyline) {

						$self->in_transit->set_polyline(

							uid      => $uid,

							db       => $db,

							polyline => $polyline,

						);

					}



					# mustn't be called during a transaction

					if ( not $opt{in_transaction} ) {

						$self->run_hook( $uid, 'checkin' );

					}



					$promise->resolve($trip);

				}

			)->catch(

				sub {

					my ($err) = @_;

					$promise->reject($err);

					return;

				}

			)->wait;



			return $promise;

		}

	);



	$self->helper(

		'_checkin_dbris_p' => sub {

			my ( $self, %opt ) = @_;
@@ -966,7 +1124,7 @@ sub startup { 
				return $promise->resolve( 0, 'race condition' );

			}



			if ( $user->{is_dbris} or $user->{is_hafas} ) {

			if ( $user->{is_dbris} or $user->{is_hafas} or $user->{is_motis} ) {

				return $self->_checkout_journey_p(%opt);

			}
@@ -2026,6 +2184,7 @@ sub startup { 
					is_dbris        => $latest->{is_dbris},

					is_iris         => $latest->{is_iris},

					is_hafas        => $latest->{is_hafas},

					is_motis        => $latest->{is_motis},

					journey_id      => $latest->{journey_id},

					timestamp       => $action_time,

					timestamp_delta => $now->epoch - $action_time->epoch,
@@ -2033,10 +2192,12 @@ sub startup { 
					train_line      => $latest->{train_line},

					train_no        => $latest->{train_no},

					train_id        => $latest->{train_id},

					train_color     => $latest->{train_color},

					sched_departure => epoch_to_dt( $latest->{sched_dep_ts} ),

					real_departure  => epoch_to_dt( $latest->{real_dep_ts} ),

					dep_ds100       => $latest->{dep_ds100},

					dep_eva         => $latest->{dep_eva},

					dep_external_id => $latest->{dep_external_id},

					dep_name        => $latest->{dep_name},

					dep_lat         => $latest->{dep_lat},

					dep_lon         => $latest->{dep_lon},
@@ -2045,6 +2206,7 @@ sub startup { 
					real_arrival    => epoch_to_dt( $latest->{real_arr_ts} ),

					arr_ds100       => $latest->{arr_ds100},

					arr_eva         => $latest->{arr_eva},

					arr_external_id => $latest->{arr_external_id},

					arr_name        => $latest->{arr_name},

					arr_lat         => $latest->{arr_lat},

					arr_lon         => $latest->{arr_lon},
@@ -2088,6 +2250,7 @@ sub startup { 
					id => $status->{backend_id},

					type => $status->{is_dbris} ? 'DBRIS'

					: $status->{is_hafas} ? 'HAFAS'

					: $status->{is_motis} ? 'MOTIS'

					: 'IRIS-TTS',

					name => $status->{backend_name},

				},
@@ -2124,6 +2287,7 @@ sub startup { 
					line    => $status->{train_line},

					no      => $status->{train_no},

					id      => $status->{train_id},

					color   => $status->{train_color},

					hafasId => $status->{extra_data}{trip_id},

				},

				intermediateStops => [],

lib/Travelynx/Command/database.pm

+226 −2
Original line number Diff line number Diff line 
package Travelynx::Command::database;



# Copyright (C) 2020-2023 Birte Kristina Friesel

# Copyright (C) 2025 networkException <git@nwex.de>

#

# SPDX-License-Identifier: AGPL-3.0-or-later

use Mojo::Base 'Mojolicious::Command';
@@ -11,6 +12,7 @@ use List::Util qw(); 
use JSON;

use Travel::Status::DE::HAFAS;

use Travel::Status::DE::IRIS::Stations;

use Travel::Status::MOTIS;



has description => 'Initialize or upgrade database layout';
@@ -2854,6 +2856,187 @@ qq{select distinct checkout_station_id from in_transit where backend_id = 0;} 
			}

		);

	},



	# v61 -> v62

	# Add MOTIS backend type, add RNV and transitous MOTIS backends

	sub {

		my ($db) = @_;

		$db->query(

			qq{

				alter table backends add column motis bool default false;

				alter table schema_version add column motis varchar(12);



				drop view users_with_backend;

				create view users_with_backend as select

					users.id as id, users.name as name, status, public_level,

					email, password, registered_at, last_seen,

					deletion_requested, deletion_notified, use_history,

					accept_follows, notifications, profile, backend_id, iris,

					hafas, efa, dbris, motis, backend.name as backend_name

					from users

					left join backends as backend on users.backend_id = backend.id

					;



				create table stations_external_ids (

					eva serial not null primary key,

					backend_id smallint not null,

					external_id text not null,



					unique (backend_id, external_id),

					foreign key (eva, backend_id) references stations (eva, source)

				);



				create view stations_with_external_ids as select

					stations.*, stations_external_ids.external_id

					from stations

					left join stations_external_ids on

						stations.eva = stations_external_ids.eva and

						stations.source = stations_external_ids.backend_id

					;



				alter table in_transit add column train_color varchar(6);

				alter table journeys add column train_color varchar(6);



				drop view in_transit_str;

				drop view journeys_str;

				drop view users_with_backend;

				drop view follows_in_transit;



				create view in_transit_str as select

					user_id,

					backend.iris as is_iris, backend.hafas as is_hafas,

					backend.efa as is_efa, backend.dbris as is_dbris,

					backend.motis as is_motis,

					backend.name as backend_name, in_transit.backend_id as backend_id,

					train_type, train_line, train_no, train_id, train_color,

					extract(epoch from checkin_time) as checkin_ts,

					extract(epoch from sched_departure) as sched_dep_ts,

					extract(epoch from real_departure) as real_dep_ts,

					checkin_station_id as dep_eva,

					dep_station.ds100 as dep_ds100,

					dep_station.name as dep_name,

					dep_station.lat as dep_lat,

					dep_station.lon as dep_lon,

					dep_station_external_id.external_id as dep_external_id,

					extract(epoch from checkout_time) as checkout_ts,

					extract(epoch from sched_arrival) as sched_arr_ts,

					extract(epoch from real_arrival) as real_arr_ts,

					checkout_station_id as arr_eva,

					arr_station.ds100 as arr_ds100,

					arr_station.name as arr_name,

					arr_station.lat as arr_lat,

					arr_station.lon as arr_lon,

					arr_station_external_id.external_id as arr_external_id,

					polyline_id,

					polylines.polyline as polyline,

					visibility,

					coalesce(visibility, users.public_level & 127) as effective_visibility,

					cancelled, route, messages, user_data,

					dep_platform, arr_platform, data

					from in_transit

					left join polylines on polylines.id = polyline_id

					left join users on users.id = user_id

					left join stations as dep_station on checkin_station_id = dep_station.eva and in_transit.backend_id = dep_station.source

					left join stations as arr_station on checkout_station_id = arr_station.eva and in_transit.backend_id = arr_station.source

					left join stations_external_ids as dep_station_external_id on checkin_station_id = dep_station_external_id.eva and in_transit.backend_id = dep_station_external_id.backend_id

					left join stations_external_ids as arr_station_external_id on checkout_station_id = arr_station_external_id.eva and in_transit.backend_id = arr_station_external_id.backend_id

					left join backends as backend on in_transit.backend_id = backend.id

					;

				create view journeys_str as select

					journeys.id as journey_id, user_id,

					backend.iris as is_iris, backend.hafas as is_hafas,

					backend.efa as is_efa, backend.dbris as is_dbris,

					backend.motis as is_motis,

					backend.name as backend_name, journeys.backend_id as backend_id,

					train_type, train_line, train_no, train_id, train_color,

					extract(epoch from checkin_time) as checkin_ts,

					extract(epoch from sched_departure) as sched_dep_ts,

					extract(epoch from real_departure) as real_dep_ts,

					checkin_station_id as dep_eva,

					dep_station.ds100 as dep_ds100,

					dep_station.name as dep_name,

					dep_station.lat as dep_lat,

					dep_station.lon as dep_lon,

					dep_station_external_id.external_id as dep_external_id,

					extract(epoch from checkout_time) as checkout_ts,

					extract(epoch from sched_arrival) as sched_arr_ts,

					extract(epoch from real_arrival) as real_arr_ts,

					checkout_station_id as arr_eva,

					arr_station.ds100 as arr_ds100,

					arr_station.name as arr_name,

					arr_station.lat as arr_lat,

					arr_station.lon as arr_lon,

					arr_station_external_id.external_id as arr_external_id,

					polylines.polyline as polyline,

					visibility,

					coalesce(visibility, users.public_level & 127) as effective_visibility,

					cancelled, edited, route, messages, user_data,

					dep_platform, arr_platform

					from journeys

					left join polylines on polylines.id = polyline_id

					left join users on users.id = user_id

					left join stations as dep_station on checkin_station_id = dep_station.eva and journeys.backend_id = dep_station.source

					left join stations as arr_station on checkout_station_id = arr_station.eva and journeys.backend_id = arr_station.source

					left join stations_external_ids as dep_station_external_id on checkin_station_id = dep_station_external_id.eva and journeys.backend_id = dep_station_external_id.backend_id

					left join stations_external_ids as arr_station_external_id on checkout_station_id = arr_station_external_id.eva and journeys.backend_id = arr_station_external_id.backend_id

					left join backends as backend on journeys.backend_id = backend.id

					;

				create view users_with_backend as select

					users.id as id, users.name as name, status, public_level,

					email, password, registered_at, last_seen,

					deletion_requested, deletion_notified, use_history,

					accept_follows, notifications, profile, backend_id, iris,

					hafas, efa, dbris, motis, backend.name as backend_name

					from users

					left join backends as backend on users.backend_id = backend.id

					;

				create view follows_in_transit as select

					r1.subject_id as follower_id, user_id as followee_id,

					users.name as followee_name,

					train_type, train_line, train_no, train_id, train_color,

					backend.iris as is_iris, backend.hafas as is_hafas,

					backend.efa as is_efa, backend.dbris as is_dbris,

					backend.motis as is_motis,

					backend.name as backend_name, in_transit.backend_id as backend_id,

					extract(epoch from checkin_time) as checkin_ts,

					extract(epoch from sched_departure) as sched_dep_ts,

					extract(epoch from real_departure) as real_dep_ts,

					checkin_station_id as dep_eva,

					dep_station.ds100 as dep_ds100,

					dep_station.name as dep_name,

					dep_station.lat as dep_lat,

					dep_station.lon as dep_lon,

					extract(epoch from checkout_time) as checkout_ts,

					extract(epoch from sched_arrival) as sched_arr_ts,

					extract(epoch from real_arrival) as real_arr_ts,

					checkout_station_id as arr_eva,

					arr_station.ds100 as arr_ds100,

					arr_station.name as arr_name,

					arr_station.lat as arr_lat,

					arr_station.lon as arr_lon,

					polyline_id,

					polylines.polyline as polyline,

					visibility,

					coalesce(visibility, users.public_level & 127) as effective_visibility,

					cancelled, route, messages, user_data,

					dep_platform, arr_platform, data

					from in_transit

					left join polylines on polylines.id = polyline_id

					left join users on users.id = user_id

					left join relations as r1 on r1.predicate = 1 and r1.object_id = user_id

					left join stations as dep_station on checkin_station_id = dep_station.eva and in_transit.backend_id = dep_station.source

					left join stations as arr_station on checkout_station_id = arr_station.eva and in_transit.backend_id = arr_station.source

					left join backends as backend on in_transit.backend_id = backend.id

					order by checkin_time desc

					;

			}

		);

		$db->query(

			qq{

				update schema_version set version = 62;

			}

		);

	},

);



sub sync_stations {
@@ -3044,7 +3227,7 @@ sub sync_stations { 
	}

}



sub sync_backends {

sub sync_backends_hafas {

	my ($db) = @_;

	for my $service ( Travel::Status::DE::HAFAS::get_services() ) {

		my $present = $db->select(
@@ -3074,6 +3257,36 @@ sub sync_backends { 
		{ hafas => $Travel::Status::DE::HAFAS::VERSION } );

}



sub sync_backends_motis {

	my ($db) = @_;

	for my $service ( Travel::Status::MOTIS::get_services() ) {

		my $present = $db->select(

			'backends',

			'count(*) as count',

			{

				motis => 1,

				name  => $service->{shortname}

			}

		)->hash->{count};

		if ( not $present ) {

			$db->insert(

				'backends',

				{

					iris  => 0,

					hafas => 0,

					efa   => 0,

					dbris => 0,

					motis => 1,

					name  => $service->{shortname},

				},

				{ on_conflict => undef }

			);

		}

	}



	$db->update( 'schema_version', { motis => $Travel::Status::MOTIS::VERSION } );

}



sub setup_db {

	my ($db) = @_;

	my $tx = $db->begin;
@@ -3169,7 +3382,18 @@ sub migrate_db { 
	else {

		say

"Synchronizing with Travel::Status::DE::HAFAS $Travel::Status::DE::HAFAS::VERSION";

		sync_backends($db);

		sync_backends_hafas($db);

	}



	my $motis_version = get_schema_version( $db, 'motis' ) // '0';

	say "Found backend table for Motis v${motis_version}";

	if ( $motis_version eq $Travel::Status::MOTIS::VERSION ) {

		say 'Backend table is up-to-date';

	}

	else {

		say

"Synchronizing with Travel::Status::MOTIS $Travel::Status::MOTIS::VERSION";

		sync_backends_motis($db);

	}



	$db->update( 'schema_version',

lib/Travelynx/Command/work.pm

+95 −0
Original line number Diff line number Diff line 
package Travelynx::Command::work;



# Copyright (C) 2020-2023 Birte Kristina Friesel

# Copyright (C) 2025 networkException <git@nwex.de>

#

# SPDX-License-Identifier: AGPL-3.0-or-later

use Mojo::Base 'Mojolicious::Command';
@@ -172,6 +173,100 @@ sub run { 
			next;

		}



		if ( $entry->{is_motis} ) {



			eval {

				$self->app->motis->trip_id(

					service => $entry->{backend_name},

					trip_id => $train_id,

				)->then(

					sub {

						my ($journey) = @_;



						for my $stopover ( $journey->stopovers ) {

							if ( not defined $stopover->stop->{eva} ) {

								my $stop = $self->app->stations->get_by_external_id(

									external_id => $stopover->stop->id,

									motis       => $entry->{backend_name},

								);



								$stopover->stop->{eva} = $stop->{eva};

							}

						}



						my $found_departure;

						my $found_arrival;

						for my $stopover ( $journey->stopovers ) {

							if ( $stopover->stop->{eva} == $dep ) {

								$found_departure = $stopover;

							}



							if ( $arr and $stopover->stop->{eva} == $arr ) {

								$found_arrival = $stopover;

								last;

							}

						}



						if ( not $found_departure ) {

							$self->app->log->debug("Did not find $dep within trip $train_id");

							return;

						}



						if ( $found_departure->realtime_departure ) {

							$self->app->in_transit->update_departure_motis(

								uid      => $uid,

								journey  => $journey,

								stopover => $found_departure,

								dep_eva  => $dep,

								arr_eva  => $arr,

								train_id => $train_id,

							);

						}



						if ( $found_arrival and $found_arrival->realtime_arrival ) {

							$self->app->in_transit->update_arrival_motis(

								uid      => $uid,

								journey  => $journey,

								train_id => $train_id,

								stopover => $found_arrival,

								dep_eva  => $dep,

								arr_eva  => $arr

							);

						}

					}

				)->catch(

					sub {

						my ($err) = @_;

						$self->app->log->error(

"work($uid) @ MOTIS $entry->{backend_name}: journey: $err"

						);

						if ( $err =~ m{HTTP 429} ) {

							$dbris_rate_limited = 1;

						}

					}

				)->wait;



				if (    $arr

					and $entry->{real_arr_ts}

					and $now->epoch - $entry->{real_arr_ts} > 600 )

				{

					$self->app->checkout_p(

						station => $arr,

						force   => 2,

						dep_eva => $dep,

						arr_eva => $arr,

						uid     => $uid

					)->wait;

				}

			};

			if ($@) {

				$errors += 1;

				$self->app->log->error(

					"work($uid) @ MOTIS $entry->{backend_name}: $@");

			}

			next;

		}



		if ( $entry->{is_hafas} ) {



			eval {

lib/Travelynx/Controller/Account.pm

+47 −0
Original line number Diff line number Diff line 
package Travelynx::Controller::Account;



# Copyright (C) 2020-2023 Birte Kristina Friesel

# Copyright (C) 2025 networkException <git@nwex.de>

#

# SPDX-License-Identifier: AGPL-3.0-or-later

use Mojo::Base 'Mojolicious::Controller';
@@ -1137,6 +1138,52 @@ sub backend_form { 
				$type = undef;

			}

		}

		elsif ( $backend->{motis} ) {

			my $s = $self->motis->get_service( $backend->{name} );



			$type                = 'MOTIS';

			$backend->{longname} = $s->{name};

			$backend->{homepage} = $s->{homepage};

			$backend->{regions}  = [ map { $place_map{$_} // $_ }

					@{ $s->{coverage}{regions} // [] } ];

			$backend->{has_area} = $s->{coverage}{area} ? 1 : 0;



			if ( $backend->{name} eq 'transitous' ) {

				$backend->{regions} = [ 'Weltweit' ];

			}

			if ( $backend->{name} eq 'RNV' ) {

				$backend->{homepage} = 'https://rnv-online.de/';

			}



			if (

					$s->{coverage}{area}

				and $s->{coverage}{area}{type} eq 'Polygon'

				and $self->lonlat_in_polygon(

					$s->{coverage}{area}{coordinates},

					[ $user_lon, $user_lat ]

				)

				)

			{

				push( @suggested_backends, $backend );

			}

			elsif ( $s->{coverage}{area}

				and $s->{coverage}{area}{type} eq 'MultiPolygon' )

			{

				for my $s_poly (

					@{ $s->{coverage}{area}{coordinates} // [] } )

				{

					if (

						$self->lonlat_in_polygon(

							$s_poly, [ $user_lon, $user_lat ]

						)

						)

					{

						push( @suggested_backends, $backend );

						last;

					}

				}

			}

		}

		$backend->{type} = $type;

	}

lib/Travelynx/Controller/Api.pm

+2 −0
Original line number Diff line number Diff line
@@ -189,6 +189,7 @@ sub travel_v1 { 
		my $train_id;

		my $dbris = sanitize( undef, $payload->{dbris} );

		my $hafas = sanitize( undef, $payload->{hafas} );

		my $motis = sanitize( undef, $payload->{motis} );



		if ( not $hafas and exists $payload->{train}{journeyID} ) {

			$dbris //= 'bahn.de';
@@ -298,6 +299,7 @@ sub travel_v1 { 
					uid      => $uid,

					hafas    => $hafas,

					dbris    => $dbris,

					motis    => $motis,

				);

			}

		)->then(