Unverified Commit 28db8ddf authored by Birte Kristina Friesel's avatar Birte Kristina Friesel
Browse files

Use background worker for (slow) GDPR export

parent 519e9987

.gitignore

+4 −1
Original line number Diff line number Diff line 
# Deployment-specific configuration

/travelynx.conf



# Depndencies (when using carton/cpanm)

# Dependencies (when using carton/cpanm)

/local



# Temporary data, e.g. GDPR exports

/public/tmp/*.json



# Created by https://www.gitignore.io/api/vim,perl

# Edit at https://www.gitignore.io/?templates=vim,perl

lib/Travelynx.pm

+5 −0
Original line number Diff line number Diff line
@@ -3228,6 +3228,9 @@ sub startup { 
	  ->to('profile#journey_details');

	$r->get('/.well-known/webfinger')->to('account#webfinger');

	$r->get('/dyn/:av/autocomplete.js')->to('api#autocomplete');

	$r->any('/tmp/*whatever' => sub {

		shift->render('not_found', status => 404);

	});

	$r->post('/api/v1/import')->to('api#import_v1');

	$r->post('/api/v1/travel')->to('api#travel_v1');

	$r->post('/action')->to('traveling#travel_action');
@@ -3284,6 +3287,7 @@ sub startup { 
	  ->to( 'traveling#cancelled', format => undef );

	$authed_r->get('/checkin/add')->to('traveling#add_intransit_form');

	$authed_r->get('/fgr')->to('passengerrights#list_candidates');

	$authed_r->get('/account/download-export')->to('account#download_export')->name('account_download_export');

	$authed_r->get('/account/password')->to('account#password_form');

	$authed_r->get('/account/mail')->to('account#change_mail');

	$authed_r->get('/account/name')->to('account#change_name');
@@ -3321,6 +3325,7 @@ sub startup { 
	$authed_r->post('/account/insight')->to('account#insight');

	$authed_r->post('/account/language')->to('account#change_language');

	$authed_r->post('/account/select_backend')->to('account#change_backend');

	$authed_r->post('/account/export')->to('account#export')->name('account_export');

	$authed_r->post('/checkin/add')->to('traveling#add_intransit_form');

	$authed_r->post('/journey/add')->to('traveling#add_journey_form');

	$authed_r->post('/polyline/set')->to('traveling#set_polyline');

lib/Travelynx/Command/database.pm

+24 −0
Original line number Diff line number Diff line
@@ -3819,6 +3819,30 @@ qq{select distinct checkout_station_id from in_transit where backend_id = 0;} 
			}

		);

	},



	# v73 -> v74

	# Use asynchronous background worker for data export

	sub {

		my ($db) = @_;

		$db->query(

			qq{

				drop view users_with_backend;

				alter table users

					add column export_requested smallint,

					add column export_filename varchar(64);

				create view users_with_backend as select

					users.id as id, users.name as name, status, public_level,

					language, email, password, registered_at, last_seen,

					deletion_requested, deletion_notified, use_history,

					accept_follows, notifications, profile, backend_id, iris,

					hafas, efa, dbris, motis, backend.name as backend_name

					from users

					left join backends as backend on users.backend_id = backend.id

					;

				update schema_version set version = 74;

			}

		);

	},

);



sub sync_dbdb {

lib/Travelynx/Command/work.pm

+48 −1
Original line number Diff line number Diff line
@@ -10,6 +10,7 @@ use Mojo::Promise; 
use utf8;



use DateTime;

use File::Slurp qw(write_file);

use JSON;

use List::Util;
@@ -22,7 +23,6 @@ sub run { 


	my $now              = DateTime->now( time_zone => 'Europe/Berlin' );

	my $checkin_deadline = $now->clone->subtract( hours => 48 );

	my $json             = JSON->new;



	if ( -e 'maintenance' ) {

		$self->app->log->debug('work: "maintenance" file found, aborting');
@@ -877,6 +877,53 @@ sub run { 


	}



	if ( not $backend or $backend eq 'export-user-data' ) {

		my $db           = $self->app->pg->db;

		my $json         = JSON->new->utf8;

		my $now_yyyymmdd = $now->strftime('%Y%m%d');

		for my $uid ( $self->app->users->get_export_requests ) {

			my $user_data = {

				account =>

				  $db->select( 'users', '*', { id => $uid } )->expand->hash,

				backends => [ $self->app->stations->get_backends( db => $db ) ],

				in_transit => [

					$db->select( 'in_transit_str', '*', { user_id => $uid } )

					  ->expand->hashes->each

				],

				journeys => [

					$db->select( 'journeys_str', '*', { user_id => $uid } )

					  ->expand->hashes->each

				],

			};

			my $name     = $user_data->{account}{name};

			my $filename = "travelynx-export-${name}-${now_yyyymmdd}.json";



			write_file( "public/tmp/$filename", $json->encode($user_data) );



			$self->app->users->set_export(

				uid      => $uid,

				status   => 2,

				filename => $filename,

			);

		}

		for my $res ( $self->app->users->get_export_filenames ) {

			my $uid      = $res->{id};

			my $filename = $res->{export_filename};

			if ( $filename =~ m{ (\d{8}) [.] json $ }x ) {

				my $yyyymmdd = $1;

				if ( $now_yyyymmdd - $yyyymmdd >= 2 ) {

					unlink("public/tmp/${filename}")

					  or warn("could not unlink public/tmp/${filename}: $!");

					$self->app->users->set_export(

						uid      => $uid,

						status   => 0,

						filename => undef

					);

				}

			}

		}

	}



	my $started_at       = $now;

	my $main_finished_at = DateTime->now( time_zone => 'Europe/Berlin' );

	my $worker_duration  = $main_finished_at->epoch - $started_at->epoch;

lib/Travelynx/Controller/Account.pm

+50 −0
Original line number Diff line number Diff line
@@ -733,6 +733,52 @@ sub social_action { 
	$self->redirect_to($redirect_to);

}



sub download_export {

	my ($self) = @_;

	my $user = $self->current_user;

	my ( $export_status, $export_filename )

	  = $self->users->get_export( uid => $user->{id} );



	if ( $export_status == 2 ) {

		$self->res->headers->content_disposition(

			"attachment; filename=${export_filename};");

		$self->reply->static("tmp/${export_filename}");

	}

	else {

		$self->redirect_to('/account');

	}



}



sub export {

	my ($self) = @_;

	my $user = $self->current_user;



	if ( $self->validation->csrf_protect->has_error('csrf_token') ) {

		$self->render(

			'bad_request',

			csrf   => 1,

			status => 400

		);

		return;

	}



	if ( $self->param('action') and $self->param('action') eq 'export' ) {

		my ( $export_status, $export_filename )

		  = $self->users->get_export( uid => $user->{id} );

		if ($export_filename) {

			unlink("public/tmp/${export_filename}");

		}

		$self->users->set_export(

			uid      => $user->{id},

			status   => 1,

			filename => undef,

		);

		$self->flash( success => 'export' );

	}

	$self->redirect_to('account');

}



sub profile {

	my ($self) = @_;

	my $user = $self->current_user;
@@ -1548,6 +1594,8 @@ sub account { 
	my $followers = $self->users->has_followers( uid => $uid );

	my $following = $self->users->has_followees( uid => $uid );

	my $blocked   = $self->users->has_blocked_users( uid => $uid );

	my ( $export_status, $export_filename )

	  = $self->users->get_export( uid => $uid );



	$self->render(

		'account',
@@ -1557,6 +1605,8 @@ sub account { 
		num_followers          => $followers,

		num_following          => $following,

		num_blocked            => $blocked,

		export_status          => $export_status,

		export_filename        => $export_filename,

	);

	$self->users->mark_seen( uid => $uid );

}