Unverified Commit 2f9ba6e0 authored by Birte Kristina Friesel's avatar Birte Kristina Friesel
Browse files

switch to internal station database; add out-of-service stations for old journeys

parent d7918251

README.md

+2 −1
Original line number Diff line number Diff line
@@ -31,7 +31,7 @@ however this method is untested. 
In the project root directory (where `cpanfile` resides), run



```

carton install

carton install --deployment

```



and set `PERL5LIB=.../local/lib/perl5` before executing any travelynx
@@ -87,6 +87,7 @@ or not. 


```

git pull

carton install --deployment # if you are using carton: update dependencies

chmod -R a+rX . # only needed if travelynx is running under a different user

if perl index.pl database has-current-schema; then

    systemctl reload travelynx

lib/Travelynx.pm

+45 −87
Original line number Diff line number Diff line
@@ -20,7 +20,6 @@ use List::Util; 
use List::UtilsBy   qw(uniq_by);

use List::MoreUtils qw(first_index);

use Travel::Status::DE::DBWagenreihung;

use Travel::Status::DE::IRIS::Stations;

use Travelynx::Helper::DBDB;

use Travelynx::Helper::HAFAS;

use Travelynx::Helper::IRIS;
@@ -29,6 +28,7 @@ use Travelynx::Helper::Traewelling; 
use Travelynx::Model::InTransit;

use Travelynx::Model::Journeys;

use Travelynx::Model::JourneyStatsCache;

use Travelynx::Model::Stations;

use Travelynx::Model::Traewelling;

use Travelynx::Model::Users;
@@ -55,27 +55,6 @@ sub epoch_to_dt { 
	);

}



sub get_station {

	my ( $station_name, $exact_match ) = @_;



	my @candidates

	  = Travel::Status::DE::IRIS::Stations::get_station($station_name);



	if ( @candidates == 1 ) {

		if ( not $exact_match ) {

			return $candidates[0];

		}

		if (   $candidates[0][0] eq $station_name

			or $candidates[0][1] eq $station_name

			or $candidates[0][2] eq $station_name )

		{

			return $candidates[0];

		}

		return undef;

	}

	return undef;

}



sub startup {

	my ($self) = @_;
@@ -227,19 +206,11 @@ sub startup { 
	$self->attr(

		coordinates_by_station => sub {

			my $legacy_names = $self->app->renamed_station;

			my %location;

			for

			  my $station ( Travel::Status::DE::IRIS::Stations::get_stations() )

			{

				if ( $station->[3] ) {

					$location{ $station->[1] }

					  = [ $station->[4], $station->[3] ];

				}

			}

			my $location     = $self->stations->get_latlon_by_name;

			while ( my ( $old_name, $new_name ) = each %{$legacy_names} ) {

				$location{$old_name} = $location{$new_name};

				$location->{$old_name} = $location->{$new_name};

			}

			return \%location;

			return $location;

		}

	);
@@ -261,18 +232,6 @@ sub startup { 
		}

	);



	$self->attr(

		station_by_eva => sub {

			my %map;

			for

			  my $station ( Travel::Status::DE::IRIS::Stations::get_stations() )

			{

				$map{ $station->[2] } = $station;

			}

			return \%map;

		}

	);



	if ( not $self->app->config->{base_url} ) {

		$self->app->log->error(

"travelynx.conf: 'base_url' is missing. Links in maintenance/work/worker-generated E-Mails will be incorrect. This variable was introduced in travelynx 1.22; see examples/travelynx.conf for documentation."
@@ -369,7 +328,7 @@ sub startup { 
				in_transit      => $self->in_transit,

				stats_cache     => $self->journey_stats_cache,

				renamed_station => $self->app->renamed_station,

				station_by_eva  => $self->app->station_by_eva,

				stations        => $self->stations,

			);

		}

	);
@@ -409,6 +368,14 @@ sub startup { 
		}

	);



	$self->helper(

		stations => sub {

			my ($self) = @_;

			state $stations

			  = Travelynx::Model::Stations->new( pg => $self->pg );

		}

	);



	$self->helper(

		users => sub {

			my ($self) = @_;
@@ -457,7 +424,8 @@ sub startup { 


			my @unknown_stations;

			for my $station (@stations) {

				my $station_info = get_station($station);

				my $station_info

				  = $self->stations->get_by_name( $station );

				if ( not $station_info ) {

					push( @unknown_stations, $station );

				}
@@ -689,7 +657,7 @@ sub startup { 
				($train) = List::Util::first { $_->train_id eq $train_id }

				@{ $status->{results} };

				if (    $train

					and $self->app->station_by_eva->{ $train->station_uic } )

					and $self->stations->get_by_eva( $train->station_uic ) )

				{

					$new_checkout_station_id = $train->station_uic;

				}
@@ -1426,17 +1394,17 @@ sub startup { 
			if ($in_transit) {



				if ( my $station

					= $self->app->station_by_eva->{ $in_transit->{dep_eva} } )

					= $self->stations->get_by_eva( $in_transit->{dep_eva} ) )

				{

					$in_transit->{dep_ds100} = $station->[0];

					$in_transit->{dep_name}  = $station->[1];

					$in_transit->{dep_ds100} = $station->{ds100};

					$in_transit->{dep_name}  = $station->{name};

				}

				if ( $in_transit->{arr_eva}

					and my $station

					= $self->app->station_by_eva->{ $in_transit->{arr_eva} } )

					= $self->stations->get_by_eva( $in_transit->{arr_eva} ) )

				{

					$in_transit->{arr_ds100} = $station->[0];

					$in_transit->{arr_name}  = $station->[1];

					$in_transit->{arr_ds100} = $station->{ds100};

					$in_transit->{arr_name}  = $station->{name};

				}



				my @route = @{ $in_transit->{route} // [] };
@@ -1664,22 +1632,22 @@ sub startup { 


			if ( $latest_cancellation and $latest_cancellation->{cancelled} ) {

				if (

					my $station = $self->app->station_by_eva->{

					my $station = $self->stations->get_by_eva(

						$latest_cancellation->{dep_eva}

					}

					)

				  )

				{

					$latest_cancellation->{dep_ds100} = $station->[0];

					$latest_cancellation->{dep_name}  = $station->[1];

					$latest_cancellation->{dep_ds100} = $station->{ds100};

					$latest_cancellation->{dep_name}  = $station->{name};

				}

				if (

					my $station = $self->app->station_by_eva->{

					my $station = $self->stations->get_by_eva(

						$latest_cancellation->{arr_eva}

					}

					)

				  )

				{

					$latest_cancellation->{arr_ds100} = $station->[0];

					$latest_cancellation->{arr_name}  = $station->[1];

					$latest_cancellation->{arr_ds100} = $station->{ds100};

					$latest_cancellation->{arr_name}  = $station->{name};

				}

			}

			else {
@@ -1690,16 +1658,16 @@ sub startup { 
				my $ts          = $latest->{checkout_ts};

				my $action_time = epoch_to_dt($ts);

				if ( my $station

					= $self->app->station_by_eva->{ $latest->{dep_eva} } )

					= $self->stations->get_by_eva( $latest->{dep_eva} ) )

				{

					$latest->{dep_ds100} = $station->[0];

					$latest->{dep_name}  = $station->[1];

					$latest->{dep_ds100} = $station->{ds100};

					$latest->{dep_name}  = $station->{name};

				}

				if ( my $station

					= $self->app->station_by_eva->{ $latest->{arr_eva} } )

					= $self->stations->get_by_eva( $latest->{arr_eva} ) )

				{

					$latest->{arr_ds100} = $station->[0];

					$latest->{arr_name}  = $station->[1];

					$latest->{arr_ds100} = $station->{ds100};

					$latest->{arr_name}  = $station->{name};

				}

				return {

					checked_in      => 0,
@@ -1816,28 +1784,18 @@ sub startup { 
			}



			if ( $status->{dep_eva} ) {

				my @station_descriptions

				  = Travel::Status::DE::IRIS::Stations::get_station(

					$status->{dep_eva} );

				if ( @station_descriptions == 1 ) {

					(

						undef, undef, undef,

						$ret->{fromStation}{longitude},

						$ret->{fromStation}{latitude}

					) = @{ $station_descriptions[0] };

				}

			}



			if ( $status->{arr_ds100} ) {

				my @station_descriptions

				  = Travel::Status::DE::IRIS::Stations::get_station(

					$status->{arr_ds100} );

				if ( @station_descriptions == 1 ) {

					(

						undef, undef, undef,

						$ret->{toStation}{longitude},

						$ret->{toStation}{latitude}

					) = @{ $station_descriptions[0] };

				if ( my $s = $self->stations->get_by_eva( $status->{dep_eva} ) )

				{

					$ret->{fromStation}{longitude} = $s->{lon};

					$ret->{fromStation}{latitude}  = $s->{lat};

				}

			}



			if ( $status->{arr_eva} ) {

				if ( my $s = $self->stations->get_by_eva( $status->{arr_eva} ) )

				{

					$ret->{toStation}{longitude} = $s->{lon};

					$ret->{toStation}{latitude}  = $s->{lat};

				}

			}

lib/Travelynx/Command/database.pm

+238 −1
Original line number Diff line number Diff line
@@ -6,12 +6,27 @@ package Travelynx::Command::database; 
use Mojo::Base 'Mojolicious::Command';



use DateTime;

use File::Slurp qw(read_file);

use JSON;

use Travel::Status::DE::IRIS::Stations;



has description => 'Initialize or upgrade database layout';



has usage => sub { shift->extract_usage };



sub get_iris_version {

	my ($db) = @_;

	my $version;



	eval { $version = $db->select( 'schema_version', ['iris'] )->hash->{iris}; };

	if ($@) {



		# If it failed, the version table does not exist -> run setup first.

		return undef;

	}

	return $version;

}



sub get_schema_version {

	my ($db) = @_;

	my $version;
@@ -1106,7 +1121,211 @@ my @migrations = ( 
			}

		);

	},



	# v26 -> v27

	# add list of stations that are not (or no longer) present in T-S-DE-IRIS

	# (in this case, stations that were removed up to 1.74)

	sub {

		my ($db) = @_;

		$db->query(

			qq{

				alter table schema_version

					add column iris varchar(12);

				create table stations (

					eva int not null primary key,

					ds100 varchar(16) not null,

					name varchar(64) not null,

					lat real not null,

					lon real not null,

					source smallint not null,

					archived bool not null

				);

				update schema_version set version = 27;

				update schema_version set iris = '0';

			}

		);

	},

);



sub sync_stations {

	my ( $db, $iris_version ) = @_;



	$db->update( 'schema_version',

		{ iris => $Travel::Status::DE::IRIS::Stations::VERSION } );



	say 'Updating stations table, this may take a while ...';

	my $total = scalar Travel::Status::DE::IRIS::Stations::get_stations();

	my $count = 0;

	for my $s ( Travel::Status::DE::IRIS::Stations::get_stations() ) {

		my ( $ds100, $name, $eva, $lon, $lat ) = @{$s};

		$db->insert(

			'stations',

			{

				eva      => $eva,

				ds100    => $ds100,

				name     => $name,

				lat      => $lat,

				lon      => $lon,

				source   => 0,

				archived => 0

			},

			{

				on_conflict => \

				  '(eva) do update set archived = false, source = 0'

			}

		);

		if ( $count++ % 1000 == 0 ) {

			printf( "    %2.0f%% complete\n", $count * 100 / $total );

		}

	}

	say '    done';



	my $res1 = $db->query(

		qq{

			select checkin_station_id

			from journeys

			left join stations on journeys.checkin_station_id = stations.eva

			where stations.eva is null

			limit 1;

		}

	)->hash;



	my $res2 = $db->query(

		qq{

			select checkout_station_id

			from journeys

			left join stations on journeys.checkout_station_id = stations.eva

			where stations.eva is null

			limit 1;

		}

	)->hash;



	if ( $res1 or $res2 ) {

		say 'Dropping stats cache for archived stations ...';

		$db->query('truncate journey_stats;');

	}



	say 'Updating archived stations ...';

	my $old_stations

	  = JSON->new->utf8->decode( scalar read_file('share/old_stations.json') );

	for my $s ( @{$old_stations} ) {

		$db->insert(

			'stations',

			{

				eva      => $s->{eva},

				ds100    => $s->{ds100},

				name     => $s->{name},

				lat      => $s->{latlong}[0],

				lon      => $s->{latlong}[1],

				source   => 0,

				archived => 1

			},

			{ on_conflict => undef }

		);

	}



	if ( $iris_version == 0 ) {

		say 'Applying EVA ID changes ...';

		for my $change (

			[ 721394, 301002, 'RKBP: Kronenplatz (U), Karlsruhe' ],

			[

				721356, 901012,

				'RKME: Ettlinger Tor/Staatstheater (U), Karlsruhe'

			],

		  )

		{

			my ( $old, $new, $desc ) = @{$change};

			my $rows = $db->update(

				'journeys',

				{ checkout_station_id => $new },

				{ checkout_station_id => $old }

			)->rows;

			$rows += $db->update(

				'journeys',

				{ checkin_station_id => $new },

				{ checkin_station_id => $old }

			)->rows;

			if ($rows) {

				say "$desc ($old -> $new) : $rows rows";

			}

		}

	}



	say 'Checking for unknown EVA IDs ...';

	my $found = 0;



	$res1 = $db->query(

		qq{

			select checkin_station_id

			from journeys

			left join stations on journeys.checkin_station_id = stations.eva

			where stations.eva is null;

		}

	);



	$res2 = $db->query(

		qq{

			select checkout_station_id

			from journeys

			left join stations on journeys.checkout_station_id = stations.eva

			where stations.eva is null;

		}

	);



	my %notified;

	while ( my $row = $res1->hash ) {

		my $eva = $row->{checkin_station_id};

		if ( not $found ) {

			$found = 1;

			say '';

			say '------------8<----------';

			say 'Travel::Status::DE::IRIS v'

			  . $Travel::Status::DE::IRIS::Stations::VERSION;

		}

		if ( not $notified{$eva} ) {

			say $eva;

			$notified{$eva} = 1;

		}

	}



	while ( my $row = $res2->hash ) {

		my $eva = $row->{checkout_station_id};

		if ( not $found ) {

			$found = 1;

			say '';

			say '------------8<----------';

			say 'Travel::Status::DE::IRIS v'

			  . $Travel::Status::DE::IRIS::Stations::VERSION;

		}

		if ( not $notified{$eva} ) {

			say $eva;

			$notified{$eva} = 1;

		}

	}



	if ($found) {

		say '------------8<----------';

		say '';

		say

'Due to a conceptual flaw in past travelynx releases, your database contains unknown EVA IDs.';

		say

'Please file a bug report titled "Missing EVA IDs after DB migration" at https://github.com/derf/travelynx/issues';

		say 'and include the list shown above in the bug report.';

		say

'If you do not have a GitHub account, please send an E-Mail to derf+travelynx@finalrewind.org instead.';

		say '';

		say 'This issue does not affect usability or long-term data integrity,';

		say 'and handling it is not time-critical.';

		say

'Past journeys referencing unknown EVA IDs may have inaccurate distance statistics,';

		say

'but this will be resolved once a future release handles those EVA IDs.';

		say 'Note that this issue was already present in previous releases.';

	}

	else {

		say 'None found.';

	}

}



sub setup_db {

	my ($db) = @_;
@@ -1129,7 +1348,7 @@ sub migrate_db { 
	say "Found travelynx schema v${schema_version}";



	if ( $schema_version == @migrations ) {

		say "Database layout is up-to-date";

		say 'Database layout is up-to-date';

	}



	eval {
@@ -1144,6 +1363,24 @@ sub migrate_db { 
		exit(1);

	}



	my $iris_version = get_iris_version($db);

	say "Found IRIS station database v${iris_version}";

	if ( $iris_version eq $Travel::Status::DE::IRIS::Stations::VERSION ) {

		say 'Station database is up-to-date';

	}

	else {

		eval {

			say

"Synchronizing with Travel::Status::DE::IRIS $Travel::Status::DE::IRIS::Stations::VERSION";

			sync_stations( $db, $iris_version );

		};

		if ($@) {

			say STDERR "Synchronization failed: $@";

			say STDERR "Rolling back to v${schema_version}";

			exit(1);

		}

	}



	if ( get_schema_version($db) == @migrations ) {

		$tx->commit;

	}

lib/Travelynx/Controller/Api.pm

+4 −22
Original line number Diff line number Diff line
@@ -7,7 +7,6 @@ use Mojo::Base 'Mojolicious::Controller'; 


use DateTime;

use List::Util;

use Travel::Status::DE::IRIS::Stations;

use UUID::Tiny qw(:std);



# Internal Helpers
@@ -184,41 +183,24 @@ sub travel_v1 { 
			return;

		}



		if (

			@{

				[

					Travel::Status::DE::IRIS::Stations::get_station(

						$from_station)

				]

			} != 1

		  )

		{

		if ( not $self->stations->search($from_station) ) {

			$self->render(

				json => {

					success    => \0,

					deprecated => \0,

					error      => 'fromStation is ambiguous',

					error      => 'Unknown fromStation',

					status     => $self->get_user_status_json_v1($uid)

				},

			);

			return;

		}



		if (

			$to_station

			and @{

				[

					Travel::Status::DE::IRIS::Stations::get_station(

						$to_station)

				]

			} != 1

		  )

		{

		if ( $to_station and not $self->stations->search($to_station) ) {

			$self->render(

				json => {

					success    => \0,

					deprecated => \0,

					error      => 'toStation is ambiguous',

					error      => 'Unknown toStation',

					status     => $self->get_user_status_json_v1($uid)

				},

			);

lib/Travelynx/Helper/IRIS.pm

+1 −0
Original line number Diff line number Diff line
@@ -13,6 +13,7 @@ use utf8; 
use Mojo::Promise;

use Mojo::UserAgent;

use Travel::Status::DE::IRIS;

use Travel::Status::DE::IRIS::Stations;



sub new {

	my ( $class, %opt ) = @_;