Unverified Commit 5356e00a authored by Birte Kristina Friesel's avatar Birte Kristina Friesel
Browse files

index.pl import stops: correctly handle MOTIS / external IDs

parent c6a9e3c2

lib/Travelynx/Command/import.pm

+48 −1
Original line number Diff line number Diff line
@@ -43,6 +43,7 @@ sub import_stops { 
	my %col;

	my $i = 0;

	my @queue;

	my @extId_queue;

	while ( my $row = $csv->getline($fh) ) {

		if ( not %col ) {

			%col = map { $row->[$_] => $_ } ( 0 .. $#{$row} );
@@ -58,6 +59,11 @@ sub import_stops { 
			my $backend_id

			  = $self->app->stations->get_backend_id(%backend_query);



			if ( $row->[ $col{extId} ] ) {

				push( @extId_queue, [ $row, $backend_id ] );

				next;

			}



			push(

				@queue,

				[
@@ -75,13 +81,51 @@ sub import_stops { 
			$self->app->stations->upsert_import(

				db         => $db,

				stations   => \@queue,

				batch_size => 100,

				batch_size => scalar @queue,

			);

			@queue = ();

		}

	}

	say "\r\e[2KImported ${i} stops";



	$i = 0;

	my $num_extIds = scalar @extId_queue;

	say "Importing ${num_extIds} stops with external IDs ...";

	for my $entry (@extId_queue) {

		my ( $row, $backend_id ) = @{$entry};

		my $s = $self->app->stations->get_by_external_id(

			db          => $db,

			external_id => $row->[ $col{extId} ],

			backend_id  => $backend_id

		);

		if ( not $s ) {

			eval {

				$self->app->stations->import_with_external_id(

					db          => $db,

					backend_id  => $backend_id,

					name        => $row->[ $col{name} ],

					eva         => $row->[ $col{id} ],

					external_id => $row->[ $col{extId} ],

					lat         => $row->[ $col{lat} ],

					lon         => $row->[ $col{lon} ],

				);

			};

			if ($@) {

				say q{};

				say STDERR 'Failed to import ' . join( q{,}, @{$row} );

				say STDERR q{};

				say STDERR $@;

				say STDERR q{};

				say STDERR 'Import aborted.';

				exit 1;

			}

		}

		if ( ++$i % 100 == 0 ) {

			printf( "\r\e[2KImporting external ID #%d (%5.1f%% done) ...",

				$i, $i * 100 / @extId_queue );

		}

	}



	close($fh);



}
@@ -114,6 +158,9 @@ sub import_journeys { 
		$backend{ $backend->{id} } = $backend;

	}



	# TODO special handling for dep_external_id / arr_external_id

	# → use IDs from database, not IDs from journey entry.



	my @journeys = sort { $a->{journey_id} <=> $b->{journey_id} }

	  @{ $import->{journeys} // [] };

lib/Travelynx/Model/Stations.pm

+22 −2
Original line number Diff line number Diff line
@@ -135,8 +135,29 @@ sub get_backends { 
	return @ret;

}



# { (db,) backend_id, name, eval, external_id, lat, lon }

sub import_with_external_id {

	my ( $self, %opt ) = @_;

	my $db = $opt{db} // $self->{pg}->db;



	$db->query(

		qq {

			with new_station as (

				insert into stations_external_ids (backend_id, external_id)

				values (?, ?)

				returning eva, backend_id

			)

			insert into stations (eva, name, lat, lon, source, archived)

			values ((select eva from new_station), ?, ?, ?, (select backend_id from new_station), ?)

		},

		(

			$opt{backend_id}, $opt{external_id}, $opt{name},

			$opt{lat},        $opt{lon},         0,

		)

	);

}



# stations => [[name, eva, lat, lon, source], ...]

# TODO motis / external_ids are not supported yet

sub upsert_import {

	my ( $self, %opt ) = @_;

	my $db         = $opt{db} // $self->{pg}->db;
@@ -295,7 +316,6 @@ sub add_or_update { 
					values (?, ?)

					returning eva, backend_id

				)



				insert into stations (eva, name, lat, lon, source, archived)

				values ((select eva from new_station), ?, ?, ?, (select backend_id from new_station), ?)

				returning *