Unverified Commit 2759d725 authored by Birte Kristina Friesel's avatar Birte Kristina Friesel
Browse files

support non-DB HAFAS backends (WiP)

parent 7811520a

lib/Travelynx.pm

+47 −19
Original line number Diff line number Diff line
@@ -448,7 +448,7 @@ sub startup { 
				return Mojo::Promise->reject('You are already checked in');

			}



			if ( $train_id =~ m{[|]} ) {

			if ( $opt{hafas} ) {

				return $self->_checkin_hafas_p(%opt);

			}
@@ -483,6 +483,8 @@ sub startup { 
							departure_eva => $eva,

							train         => $train,

							route      => [ $self->iris->route_diff($train) ],

							backend_id =>

							  $self->stations->get_backend_id( iris => 1 ),

						);

					};

					if ($@) {
@@ -530,6 +532,7 @@ sub startup { 
			my $promise = Mojo::Promise->new;



			$self->hafas->get_journey_p(

				service       => $opt{hafas},

				trip_id       => $train_id,

				with_polyline => 1

			)->then(
@@ -553,6 +556,7 @@ sub startup { 
						$self->stations->add_or_update(

							stop  => $stop,

							db    => $db,

							hafas => $opt{hafas},

						);

					}

					eval {
@@ -561,7 +565,10 @@ sub startup { 
							db         => $db,

							journey    => $journey,

							stop       => $found,

							data    => { trip_id => $journey->id }

							data       => { trip_id => $journey->id },

							backend_id => $self->stations->get_backend_id(

								hafas => $opt{hafas}

							),

						);

					};

					if ($@) {
@@ -620,8 +627,8 @@ sub startup { 
					# mustn't be called during a transaction

					if ( not $opt{in_transaction} ) {

						$self->run_hook( $uid, 'checkin' );

						if ( $journey->class <= 16 ) {

							$self->app->add_wagonorder( $uid, 1, $journey->id,

						if ( $opt{hafas} eq 'DB' and $journey->class <= 16 ) {

							$self->add_wagonorder( $uid, 1, $journey->id,

								$found->sched_dep, $journey->number );

							$self->add_stationinfo( $uid, 1, $journey->id,

								$found->loc->eva );
@@ -744,6 +751,7 @@ sub startup { 
			my $db           = $opt{db}  // $self->pg->db;

			my $user         = $self->get_user_status( $uid, $db );

			my $train_id     = $user->{train_id};

			my $hafas        = $opt{hafas};



			my $promise = Mojo::Promise->new;
@@ -765,7 +773,7 @@ sub startup { 
				return $promise->resolve( 0, 'race condition' );

			}



			if ( $train_id =~ m{[|]} ) {

			if ( $user->{is_hafas} ) {

				return $self->_checkout_hafas_p(%opt);

			}
@@ -1736,7 +1744,8 @@ sub startup { 
			if ( $latest_cancellation and $latest_cancellation->{cancelled} ) {

				if (

					my $station = $self->stations->get_by_eva(

						$latest_cancellation->{dep_eva}

						$latest_cancellation->{dep_eva},

						backend_id => $latest_cancellation->{backend_id},

					)

				  )

				{
@@ -1745,7 +1754,8 @@ sub startup { 
				}

				if (

					my $station = $self->stations->get_by_eva(

						$latest_cancellation->{arr_eva}

						$latest_cancellation->{arr_eva},

						backend_id => $latest_cancellation->{backend_id},

					)

				  )

				{
@@ -1760,14 +1770,20 @@ sub startup { 
			if ($latest) {

				my $ts          = $latest->{checkout_ts};

				my $action_time = epoch_to_dt($ts);

				if ( my $station

					= $self->stations->get_by_eva( $latest->{dep_eva} ) )

				if (

					my $station = $self->stations->get_by_eva(

						$latest->{dep_eva}, backend_id => $latest->{backend_id}

					)

				  )

				{

					$latest->{dep_ds100} = $station->{ds100};

					$latest->{dep_name}  = $station->{name};

				}

				if ( my $station

					= $self->stations->get_by_eva( $latest->{arr_eva} ) )

				if (

					my $station = $self->stations->get_by_eva(

						$latest->{arr_eva}, backend_id => $latest->{backend_id}

					)

				  )

				{

					$latest->{arr_ds100} = $station->{ds100};

					$latest->{arr_name}  = $station->{name};
@@ -1776,6 +1792,10 @@ sub startup { 
					checked_in      => 0,

					cancelled       => 0,

					cancellation    => $latest_cancellation,

					backend_id      => $latest->{backend_id},

					backend_name    => $latest->{backend_name},

					is_iris         => $latest->{is_iris},

					is_hafas        => $latest->{is_hafas},

					journey_id      => $latest->{journey_id},

					timestamp       => $action_time,

					timestamp_delta => $now->epoch - $action_time->epoch,
@@ -1834,6 +1854,11 @@ sub startup { 
					  or $status->{cancelled}

				) ? \1 : \0,

				comment => $status->{comment},

				backend => {

					id   => $status->{backend_id},

					type => $status->{is_hafas} ? 'HAFAS' : 'IRIS-TTS',

					name => $status->{backend_name},

				},

				fromStation => {

					ds100         => $status->{dep_ds100},

					name          => $status->{dep_name},
@@ -1992,6 +2017,7 @@ sub startup { 
"Eingecheckt in $traewelling->{line} nach $traewelling->{arr_name}",

						status_id => $traewelling->{status_id},

					);



					$self->traewelling->set_latest_pull_status_id(

						uid       => $uid,

						status_id => $traewelling->{status_id},
@@ -2324,6 +2350,7 @@ sub startup { 
	$authed_r->get('/account/password')->to('account#password_form');

	$authed_r->get('/account/mail')->to('account#change_mail');

	$authed_r->get('/account/name')->to('account#change_name');

	$authed_r->get('/account/select_backend')->to('account#backend_form');

	$authed_r->get('/export.json')->to('account#json_export');

	$authed_r->get('/history.json')->to('traveling#json_history');

	$authed_r->get('/history.csv')->to('traveling#csv_history');
@@ -2345,6 +2372,7 @@ sub startup { 
	$authed_r->post('/account/hooks')->to('account#webhook');

	$authed_r->post('/account/traewelling')->to('traewelling#settings');

	$authed_r->post('/account/insight')->to('account#insight');

	$authed_r->post('/account/select_backend')->to('account#change_backend');

	$authed_r->post('/journey/add')->to('traveling#add_journey_form');

	$authed_r->post('/journey/comment')->to('traveling#comment_form');

	$authed_r->post('/journey/visibility')->to('traveling#visibility_form');

lib/Travelynx/Command/database.pm

+266 −4
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ use Mojo::Base 'Mojolicious::Command'; 
use DateTime;

use File::Slurp qw(read_file);

use JSON;

use Travel::Status::DE::HAFAS;

use Travel::Status::DE::IRIS::Stations;



has description => 'Initialize or upgrade database layout';
@@ -1918,7 +1919,7 @@ my @migrations = ( 


	# v49 -> v50

	# travelynx 2.0 introduced proper HAFAS support, so there is no need for

	# the 'FYI, here is some hAFAS data' kludge anymore.

	# the 'FYI, here is some HAFAS data' kludge anymore.

	sub {

		my ($db) = @_;

		$db->query(
@@ -2310,6 +2311,235 @@ my @migrations = ( 
		);

	},



	# v54 -> v55

	# do not share stations between backends

	sub {

		my ($db) = @_;

		$db->query(

			qq{

				alter table schema_version add column hafas varchar(12);

				alter table users drop column external_services;

				alter table users add column backend_id smallint references backends (id) default 1;

				alter table stations drop constraint stations_pkey;

				alter table stations add unique (eva, source);

				create index eva_by_source on stations (eva, source);

				create index eva on stations (eva);

				alter table related_stations drop constraint related_stations_eva_meta_key;

				drop index rel_eva;

				alter table related_stations add column backend_id smallint;

				update related_stations set backend_id = 1;

				alter table related_stations alter column backend_id set not null;

				alter table related_stations add constraint backend_fk foreign key (backend_id) references backends (id);

				alter table related_stations add unique (eva, meta, backend_id);

				create index related_stations_eva_backend_key on related_stations (eva, backend_id);

			}

		);



		# up until now, IRIS and DB HAFAS shared stations, with IRIS taking

		# preference.  As of v2.7, this is no longer the case. However, old DB

		# HAFAS journeys may still reference IRIS-specific stations. So, we

		# make all IRIS stations available as DB HAFAS stations as well.

		my $total

		  = $db->select( 'stations', 'count(*) as count', { source => 0 } )

		  ->hash->{count};

		my $count = 0;



		# Caveat: If this is a fresh installation, there are no IRIS stations

		# in the database yet. So we have to populate it first.

		if ( not $total ) {

			say

'Preparing to untangle IRIS / HAFAS stations, this may take a while ...';

			$total = scalar Travel::Status::DE::IRIS::Stations::get_stations();

			for my $s ( Travel::Status::DE::IRIS::Stations::get_stations() ) {

				my ( $ds100, $name, $eva, $lon, $lat ) = @{$s};

				if ( $ENV{__TRAVELYNX_TEST_MINI_IRIS}

					and ( $eva < 8000000 or $eva > 8000100 ) )

				{

					next;

				}

				$db->insert(

					'stations',

					{

						eva      => $eva,

						ds100    => $ds100,

						name     => $name,

						lat      => $lat,

						lon      => $lon,

						source   => 0,

						archived => 0

					},

				);

				if ( $count++ % 1000 == 0 ) {

					printf( "    %2.0f%% complete\n", $count * 100 / $total );

				}

			}

			$count = 0;

		}



		say 'Untangling IRIS / HAFAS stations, this may take a while ...';

		my $res = $db->query(

			qq{

				select eva, ds100, name, lat, lon, archived

				from stations

				where source = 0;

			}

		);

		while ( my $row = $res->hash ) {

			$db->insert(

				'stations',

				{

					eva      => $row->{eva},

					ds100    => $row->{ds100},

					name     => $row->{name},

					lat      => $row->{lat},

					lon      => $row->{lon},

					archived => $row->{archived},

					source   => 1,

				}

			);

			if ( $count++ % 1000 == 0 ) {

				printf( "    %2.0f%% complete\n", $count * 100 / $total );

			}

		}

		$db->query(

			qq{

				alter table in_transit add constraint in_transit_checkin_eva_fk

					foreign key (checkin_station_id, backend_id)

					references stations (eva, source);

				alter table in_transit add constraint in_transit_checkout_eva_fk

					foreign key (checkout_station_id, backend_id)

					references stations (eva, source);

				alter table journeys add constraint journeys_checkin_eva_fk

					foreign key (checkin_station_id, backend_id)

					references stations (eva, source);

				alter table journeys add constraint journeys_checkout_eva_fk

					foreign key (checkout_station_id, backend_id)

					references stations (eva, source);

				drop view in_transit_str;

				drop view journeys_str;

				drop view follows_in_transit;

				create view in_transit_str as select

					user_id,

					backend.iris as is_iris, backend.hafas as is_hafas,

					backend.efa as is_efa, backend.ris as is_ris,

					backend.name as backend_name, in_transit.backend_id as backend_id,

					train_type, train_line, train_no, train_id,

					extract(epoch from checkin_time) as checkin_ts,

					extract(epoch from sched_departure) as sched_dep_ts,

					extract(epoch from real_departure) as real_dep_ts,

					checkin_station_id as dep_eva,

					dep_station.ds100 as dep_ds100,

					dep_station.name as dep_name,

					dep_station.lat as dep_lat,

					dep_station.lon as dep_lon,

					extract(epoch from checkout_time) as checkout_ts,

					extract(epoch from sched_arrival) as sched_arr_ts,

					extract(epoch from real_arrival) as real_arr_ts,

					checkout_station_id as arr_eva,

					arr_station.ds100 as arr_ds100,

					arr_station.name as arr_name,

					arr_station.lat as arr_lat,

					arr_station.lon as arr_lon,

					polyline_id,

					polylines.polyline as polyline,

					visibility,

					coalesce(visibility, users.public_level & 127) as effective_visibility,

					cancelled, route, messages, user_data,

					dep_platform, arr_platform, data

					from in_transit

					left join polylines on polylines.id = polyline_id

					left join users on users.id = user_id

					left join stations as dep_station on checkin_station_id = dep_station.eva and in_transit.backend_id = dep_station.source

					left join stations as arr_station on checkout_station_id = arr_station.eva and in_transit.backend_id = arr_station.source

					left join backends as backend on in_transit.backend_id = backend.id

					;

				create view journeys_str as select

					journeys.id as journey_id, user_id,

					backend.iris as is_iris, backend.hafas as is_hafas,

					backend.efa as is_efa, backend.ris as is_ris,

					backend.name as backend_name, journeys.backend_id as backend_id,

					train_type, train_line, train_no, train_id,

					extract(epoch from checkin_time) as checkin_ts,

					extract(epoch from sched_departure) as sched_dep_ts,

					extract(epoch from real_departure) as real_dep_ts,

					checkin_station_id as dep_eva,

					dep_station.ds100 as dep_ds100,

					dep_station.name as dep_name,

					dep_station.lat as dep_lat,

					dep_station.lon as dep_lon,

					extract(epoch from checkout_time) as checkout_ts,

					extract(epoch from sched_arrival) as sched_arr_ts,

					extract(epoch from real_arrival) as real_arr_ts,

					checkout_station_id as arr_eva,

					arr_station.ds100 as arr_ds100,

					arr_station.name as arr_name,

					arr_station.lat as arr_lat,

					arr_station.lon as arr_lon,

					polylines.polyline as polyline,

					visibility,

					coalesce(visibility, users.public_level & 127) as effective_visibility,

					cancelled, edited, route, messages, user_data,

					dep_platform, arr_platform

					from journeys

					left join polylines on polylines.id = polyline_id

					left join users on users.id = user_id

					left join stations as dep_station on checkin_station_id = dep_station.eva and journeys.backend_id = dep_station.source

					left join stations as arr_station on checkout_station_id = arr_station.eva and journeys.backend_id = arr_station.source

					left join backends as backend on journeys.backend_id = backend.id

					;

				create view follows_in_transit as select

					r1.subject_id as follower_id, user_id as followee_id,

					users.name as followee_name,

					train_type, train_line, train_no, train_id,

					in_transit.backend_id as backend_id,

					extract(epoch from checkin_time) as checkin_ts,

					extract(epoch from sched_departure) as sched_dep_ts,

					extract(epoch from real_departure) as real_dep_ts,

					checkin_station_id as dep_eva,

					dep_station.ds100 as dep_ds100,

					dep_station.name as dep_name,

					dep_station.lat as dep_lat,

					dep_station.lon as dep_lon,

					extract(epoch from checkout_time) as checkout_ts,

					extract(epoch from sched_arrival) as sched_arr_ts,

					extract(epoch from real_arrival) as real_arr_ts,

					checkout_station_id as arr_eva,

					arr_station.ds100 as arr_ds100,

					arr_station.name as arr_name,

					arr_station.lat as arr_lat,

					arr_station.lon as arr_lon,

					polyline_id,

					polylines.polyline as polyline,

					visibility,

					coalesce(visibility, users.public_level & 127) as effective_visibility,

					cancelled, route, messages, user_data,

					dep_platform, arr_platform, data

					from in_transit

					left join polylines on polylines.id = polyline_id

					left join users on users.id = user_id

					left join relations as r1 on r1.predicate = 1 and r1.object_id = user_id

					left join stations as dep_station on checkin_station_id = dep_station.eva and in_transit.backend_id = dep_station.source

					left join stations as arr_station on checkout_station_id = arr_station.eva and in_transit.backend_id = arr_station.source

					order by checkin_time desc

					;

				create view users_with_backend as select

					users.id as id, users.name as name, status, public_level,

					email, password, registered_at, last_seen,

					deletion_requested, deletion_notified, use_history,

					accept_follows, notifications, profile, backend_id, iris,

					hafas, efa, ris, backend.name as backend_name

					from users

					left join backends as backend on users.backend_id = backend.id

					;

				update schema_version set version = 55;

				update schema_version set hafas = '0';

			}

		);

		say

		  'This travelynx instance now has support for non-DB HAFAS backends.';

		say

'If the migration fails due to a deadlock, re-run it after stopping all background workers';

	},

);



sub sync_stations {
@@ -2341,7 +2571,7 @@ sub sync_stations { 
			},

			{

				on_conflict => \

'(eva) do update set archived = false, source = 0, ds100 = EXCLUDED.ds100, name=EXCLUDED.name, lat=EXCLUDED.lat, lon=EXCLUDED.lon'

'(eva, source) do update set archived = false, source = 0, ds100 = EXCLUDED.ds100, name=EXCLUDED.name, lat=EXCLUDED.lat, lon=EXCLUDED.lon'

			}

		);

		if ( $count++ % 1000 == 0 ) {
@@ -2500,6 +2730,26 @@ sub sync_stations { 
	}

}



sub sync_backends {

	my ($db) = @_;

	for my $service ( Travel::Status::DE::HAFAS::get_services()) {

		$db->insert(

			'backends',

			{

				iris => 0,

				hafas => 1,

				efa => 0,

				ris => 0,

				name => $service->{shortname},

			},

			{ on_conflict => undef }

		);

	}



	$db->update( 'schema_version',

		{ hafas => $Travel::Status::DE::HAFAS::VERSION } );

}



sub setup_db {

	my ($db) = @_;

	my $tx = $db->begin;
@@ -2566,9 +2816,9 @@ sub migrate_db { 
	}



	my $iris_version = get_schema_version( $db, 'iris' );

	say "Found IRIS station database v${iris_version}";

	say "Found IRIS station table v${iris_version}";

	if ( $iris_version eq $Travel::Status::DE::IRIS::Stations::VERSION ) {

		say 'Station database is up-to-date';

		say 'Station table is up-to-date';

	}

	else {

		eval {
@@ -2587,6 +2837,18 @@ sub migrate_db { 
		}

	}



	my $hafas_version = get_schema_version( $db, 'hafas' );

	say "Found backend table for HAFAS v${hafas_version}";

	if ( $hafas_version eq $Travel::Status::DE::HAFAS::VERSION ) {

		say 'Backend table is up-to-date';

	}

	else {

			say

"Synchronizing with Travel::Status::DE::HAFAS $Travel::Status::DE::HAFAS::VERSION";

		sync_backends($db);

	}





	$db->update( 'schema_version',

		{ travelynx => $self->app->config->{version} } );

lib/Travelynx/Command/dumpconfig.pm

+1 −0
Original line number Diff line number Diff line 
package Travelynx::Command::dumpconfig;



# Copyright (C) 2020-2023 Birte Kristina Friesel

#

# SPDX-License-Identifier: AGPL-3.0-or-later

lib/Travelynx/Command/work.pm

+8 −2
Original line number Diff line number Diff line
@@ -47,9 +47,12 @@ sub run { 
		my $arr      = $entry->{arr_eva};

		my $train_id = $entry->{train_id};



		if ( $train_id =~ m{[|]} ) {

		if ( $entry->{is_hafas} ) {



			$self->app->hafas->get_journey_p( trip_id => $train_id )->then(

			$self->app->hafas->get_journey_p(

				trip_id => $train_id,

				service => $entry->{backend_name}

			)->then(

				sub {

					my ($journey) = @_;
@@ -135,6 +138,9 @@ sub run { 
			next;

		}



		# TODO irgendwo ist hier ne race condition wo ein neuer checkin (in HAFAS) mit IRIS-Daten überschrieben wird.

		# Die ganzen updates brauchen wirklich mal sanity checks mit train id ...



		# Note: IRIS data is not always updated in real-time. Both departure and

		# arrival delays may take several minutes to appear, especially in case

		# of large-scale disturbances. We work around this by continuing to

lib/Travelynx/Controller/Account.pm

+46 −0
Original line number Diff line number Diff line
@@ -999,6 +999,52 @@ sub password_form { 
	$self->render('change_password');

}



sub backend_form {

	my ($self) = @_;

	my $user = $self->current_user;



	my @backends = $self->stations->get_backends;



	for my $backend (@backends) {

		my $type = 'UNKNOWN';

		if ( $backend->{iris} ) {

			$type = 'IRIS-TTS';

			$backend->{name} = 'DB';

		}

		elsif ( $backend->{hafas} ) {

			$type = 'HAFAS';

			$backend->{longname}

			  = $self->hafas->get_service( $backend->{name} )->{name};

		}

		$backend->{type} = $type;

	}



	$self->render(

		'select_backend',

		backends    => \@backends,

		user        => $user,

		redirect_to => $self->req->param('redirect_to') // '/',

	);

}



sub change_backend {

	my ($self) = @_;



	my $backend_id = $self->req->param('backend');

	my $redir      = $self->req->param('redirect_to') // '/';



	if ( $backend_id !~ m{ ^ \d+ $ }x ) {

		$self->redirect_to($redir);

	}



	$self->users->set_backend(

		uid        => $self->current_user->{id},

		backend_id => $backend_id,

	);



	$self->redirect_to($redir);

}



sub change_password {

	my ($self)       = @_;

	my $old_password = $self->req->param('oldpw');