Merge branch 'master' into production

.releaserc.json

@@ -9,17 +9,57 @@
 				"preset": "conventionalcommits",
 				"presetConfig": {
 					"types": [
-						{ "type": "feat", "hidden": false },
-						{ "type": "fix", "hidden": false },
-						{ "type": "perf", "hidden": false },
-						{ "type": "revert", "hidden": false },
-						{ "type": "docs", "hidden": true },
-						{ "type": "style", "hidden": true },
-						{ "type": "chore", "hidden": true },
-						{ "type": "refactor", "hidden": true },
-						{ "type": "test", "hidden": true },
-						{ "type": "build", "hidden": true },
-						{ "type": "ci", "hidden": true }
+						{
+							"type": "feat",
+							"section": "Features"
+						},
+						{
+							"type": "fix",
+							"section": "Bug Fixes"
+						},
+						{
+							"type": "perf",
+							"section": "Performance Improvements"
+						},
+						{
+							"type": "revert",
+							"section": "Reverts"
+						},
+						{
+							"type": "docs",
+							"section": "Documentation",
+							"hidden": true
+						},
+						{
+							"type": "style",
+							"section": "Code style",
+							"hidden": true
+						},
+						{
+							"type": "chore",
+							"section": "Others",
+							"hidden": true
+						},
+						{
+							"type": "refactor",
+							"section": "Refactoring",
+							"hidden": true
+						},
+						{
+							"type": "test",
+							"section": "Tests",
+							"hidden": true
+						},
+						{
+							"type": "build",
+							"section": "Build",
+							"hidden": true
+						},
+						{
+							"type": "ci",
+							"section": "CI",
+							"hidden": true
+						}
 					],
 					"issuePrefixes": ["ST"],
 					"issueUrlFormat": "https://perunaai.atlassian.net/browse/{{prefix}}{{id}}"

README.md

@@ -81,8 +81,7 @@ Pre scripts are processed before slave script and post scripts are processed aft
 
 > Our work is FreeBSD license, yet we sometimes use components under different licenses (e.g. Apache, GNU, CC).
 
-© 2010-2023 [CESNET](https://www.cesnet.cz/?lang=en), [CERIT-SC]
-(https://www.cerit-sc.cz/en/index.html) and [Masaryk University](https://www.muni.cz/en), all rights reserved.
+© 2010-2023 [CESNET](https://www.cesnet.cz/?lang=en), [CERIT-SC](https://www.cerit-sc.cz/en/index.html) and [Masaryk University](https://www.muni.cz/en), all rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 

gen/webcentrum_eosc

@@ -0,0 +1,12 @@
+#!/usr/bin/perl
+
+use strict;
+use warnings;
+use File::Basename;
+use perunDataGenerator;
+
+local $::SERVICE_NAME = basename($0);
+local $::PROTOCOL_VERSION = "3.0.0";
+local $::SKIP_NON_VALID_MEMBERS = 1;
+
+perunDataGenerator::generateMemberUsersDataInJson;

send/VsupIfis.pm

@@ -55,28 +55,28 @@ sub load_stag() {
 	my $db_user = $config->{"username"};
 	my $db_password = $config->{"password"};
 
-	my $dbh = DBI->connect("dbi:Pg:dbname=$db_name;host=$hostname;port=$port", $db_user, $db_password,{ RaiseError=>1, AutoCommit=>0 }) or die "Connect to database $db_name Error!\n";
+	my $dbh = DBI->connect("dbi:Oracle://$hostname:$port/$db_name", $db_user, $db_password,{ RaiseError=>1, AutoCommit=>0, LongReadLen=>65536, ora_charset => 'AL32UTF8'}) or die "Connect to database $db_name Error!\n";
+	$dbh->do("alter session set nls_date_format='YYYY-MM-DD HH24:MI:SS'");
 
 	# Select query for input database (IS/STAG) - all students with UCO_PERUN not null and STUD_DO >= now-28 or null
-	my $sth = $dbh->prepare(qq{select distinct ex_stag2idm_studia.UCO_PERUN as UCO, NS, 'STU' as TYP_VZTAHU, STUD_FORMA as DRUH_VZTAHU, ex_stag2idm_studia.ID_STUDIA as VZTAH_CISLO, STUD_FORMA as STU_FORMA, STUD_STAV as STU_STAV, STUD_TYP as STU_PROGR, STUD_OD, case when STUD_DO is not null then STUD_DO+28 ELSE STUD_DO END as STUD_DO, KARTA_LIC as KARTA_IDENT, UKONCENO as STU_GRADUATE from ex_stag2idm_studia left join ex_stag2idm_adresy on ex_stag2idm_studia.ID_STUDIA=ex_stag2idm_adresy.ID_STUDIA where ex_stag2idm_studia.UCO_PERUN is not null and (STUD_DO >= CURRENT_DATE-28 OR STUD_DO is NULL)});
+	my $sth = $dbh->prepare(qq{select distinct STUDENT_STUDIUM.UCO_PERUN as UCO, NS, 'STU' as TYP_VZTAHU, STUD_FORMA as DRUH_VZTAHU, STUDENT_STUDIUM.ID_STUDIA as VZTAH_CISLO, STUD_FORMA as STU_FORMA, STUD_STAV as STU_STAV, STUD_TYP as STU_PROGR, STUD_OD, case when STUD_DO is not null then STUD_DO+28 ELSE STUD_DO END as STUD_DO, KARTA_LIC as KARTA_IDENT, UKONCENO as STU_GRADUATE from STUDENT_STUDIUM left join STUDENT_ADRESY on STUDENT_STUDIUM.ID_STUDIA=STUDENT_ADRESY.ID_STUDIA where STUDENT_STUDIUM.UCO_PERUN is not null and (STUD_DO >= TRUNC(SYSDATE)-28 OR STUD_DO is NULL)});
 	$sth->execute();
 
 	# Structure to store data from input database (IS/STAG)
 	my $inputData = {};
 	while(my $row = $sth->fetchrow_hashref()) {
-		# disregarding the select Postgres is using lower-cased column names.
 		my $key = $row->{vztah_cislo};
-		$inputData->{$key}->{'OSB_ID'} = $row->{uco};
-		$inputData->{$key}->{'TYP_VZTAHU'} = $row->{typ_vztahu};
-		$inputData->{$key}->{'DRUH_VZTAHU'} = $row->{druh_vztahu};
-		$inputData->{$key}->{'STU_FORMA'} = $row->{stu_forma};
-		$inputData->{$key}->{'STU_STAV'} = $row->{stu_stav};
-		$inputData->{$key}->{'STU_PROGR'} = $row->{stu_progr};
-		$inputData->{$key}->{'STU_GRADUATE'} = $row->{stu_graduate};
-		$inputData->{$key}->{'NS'} = $row->{ns};
-		$inputData->{$key}->{'VZTAH_OD'} = $row->{stud_od};
-		$inputData->{$key}->{'VZTAH_DO'} = $row->{stud_do};
-		$inputData->{$key}->{'KARTA_IDENT'} = $row->{karta_ident};
+		$inputData->{$key}->{'OSB_ID'} = $row->{UCO};
+		$inputData->{$key}->{'TYP_VZTAHU'} = $row->{TYP_VZTAHU};
+		$inputData->{$key}->{'DRUH_VZTAHU'} = $row->{DRUH_VZTAHU};
+		$inputData->{$key}->{'STU_FORMA'} = $row->{STU_FORMA};
+		$inputData->{$key}->{'STU_STAV'} = $row->{STU_STAV};
+		$inputData->{$key}->{'STU_PROGR'} = $row->{STU_PROGR};
+		$inputData->{$key}->{'STU_GRADUATE'} = $row->{STU_GRADUATE};
+		$inputData->{$key}->{'NS'} = $row->{NS};
+		$inputData->{$key}->{'VZTAH_OD'} = $row->{STUD_OD};
+		$inputData->{$key}->{'VZTAH_DO'} = $row->{STUD_DO};
+		$inputData->{$key}->{'KARTA_IDENT'} = $row->{KARTA_IDENT};
 	}
 
 	# Disconnect from input database (IS/STAG)

send/generic_sender.py

@@ -95,21 +95,25 @@ def prepare_ssh_transport_command():
 			"GSSAPIAuthentication=no", "-o", "GSSAPIKeyExchange=no", "-o", "ConnectTimeout=5"]
 
 
-if __name__ == "__main__":
-	send_lib.check_input_fields(sys.argv)
-	facility_name = sys.argv[1]
-	destination = sys.argv[2]
-	if len(sys.argv) == 3:
-		# if there is no destination type, use default 'host'
-		destination_type = send_lib.DESTINATION_TYPE_HOST
-	else:
-		destination_type = sys.argv[3]
-		if destination_type == send_lib.DESTINATION_TYPE_EMAIL or destination_type == send_lib.DESTINATION_TYPE_SERVICE_SPECIFIC:
-			print("Destination type " + destination_type + " is not supported yet.", file=sys.stderr)
-			exit(1)
+def send(service_name: str, facility_name: str, destination: str, destination_type: str = send_lib.DESTINATION_TYPE_HOST, opts=None) -> None:
+	"""
+	Sends data generated for the specified service/facility to the desired destination.
+	This method runs complete generic sending logic and can be used directly from the service-specific
+	sending scripts written in python.
+	:param service_name: name of the service to send data for
+	:param facility_name: name of the facility to send data for
+	:param destination: name of the destination to send data to
+	:param destination_type: optional destination type, default is 'host'
+	:param opts: list of additional options passed to the SSH/CURL transport command
+	"""
+	if opts is None:
+		opts = []
+
+	if destination_type == send_lib.DESTINATION_TYPE_EMAIL or destination_type == send_lib.DESTINATION_TYPE_SERVICE_SPECIFIC:
+		print("Destination type " + destination_type + " is not supported yet.", file=sys.stderr)
+		exit(1)
 
 	send_lib.check_destination_format(destination, destination_type)
-	service_name = send_lib.get_global_service_name()
 
 	# choose transport command, only url type has different transport command at this moment
 	transport_command = send_lib.load_custom_transport_command(service_name)
@@ -125,6 +129,10 @@ def prepare_ssh_transport_command():
 	if port is not None:
 		transport_command.extend(["-p", port])
 
+	# Add OPTS from the service-specific send script to both SSH/cURL command
+	if (opts):
+		transport_command.extend(opts)
+
 	# add host to the transport command for all types of destination
 	transport_command.append(host)
 
@@ -194,3 +202,16 @@ def prepare_ssh_transport_command():
 						  end='')
 
 			exit(return_code)
+
+
+# Runs complete generic sending logic from the CMD line
+if __name__ == "__main__":
+	send_lib.check_input_fields(sys.argv)
+	service_name = send_lib.get_global_service_name()
+	if len(sys.argv) == 3:
+		send(service_name, sys.argv[1], sys.argv[2])
+	elif len(sys.argv) == 4:
+		send(service_name, sys.argv[1], sys.argv[2], sys.argv[3])
+	else:
+		# shouldn't happen since we check input params at start
+		send_lib.die_with_error("Wrong number of input parameters")

send/pithia_portal

@@ -1,4 +1,38 @@
-#!/bin/bash
-export SERVICE_NAME="pithia_portal"
+#!/usr/bin/env python3
+import sys
+import send_lib
+import requests
+import generic_sender
 
-python3 generic_send.py "$1" "$2" "$3"
+SERVICE_NAME = "pithia_portal"
+
+send_lib.check_input_fields(sys.argv, True)
+
+facility = sys.argv[1]
+destination = sys.argv[2]
+destination_type = sys.argv[3]
+
+send_lib.check_destination_type_allowed(destination_type, send_lib.DESTINATION_TYPE_URL)
+
+auth_token = None
+
+auth = send_lib.get_auth_credentials(SERVICE_NAME, destination)
+tokenUrl = send_lib.get_custom_config_properties(SERVICE_NAME, destination, ["tokenUrl"])[0]
+data = {
+	"username": auth[0],
+	"password": auth[1]
+}
+response = requests.post(tokenUrl, data)
+resp_cookies = response.cookies.get_dict()
+if response.status_code == 200:
+	response = response.json()
+	if "token" not in response:
+		send_lib.die_with_error("Token missing in response from token server")
+	auth_token = response["token"]
+else:
+	send_lib.die_with_error("Response from token server ended with error code: " + str(response.status_code)
+							+ ", content: " + response.text)
+
+opts = ['-H', 'Authorization: Bearer ' + auth_token, '--cookie', 'sessionid=' + resp_cookies['sessionid']]
+
+generic_sender.send(SERVICE_NAME, facility, destination, destination_type, opts)

send/umbraco_mu

@@ -18,6 +18,7 @@ send_lib.check_destination_format(destination, destination_type)
 
 auth = send_lib.get_auth_credentials(SERVICE_NAME, destination)
 
+send_lib.create_lock(SERVICE_NAME, destination)
 gen_folder = send_lib.get_gen_folder(facility, SERVICE_NAME)
 with send_lib.prepare_temporary_directory() as temp_dir:
 	send_lib.copy_files_to_directory(gen_folder, temp_dir)

send/webcentrum_eosc

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+import sys
+import requests
+import send_lib
+import tarfile
+
+SERVICE_NAME = "webcentrum_eosc"
+TIMEOUT = 1800  # 30 * 60 sec = 30 min
+
+send_lib.check_input_fields(sys.argv, True)
+
+facility = sys.argv[1]
+destination = sys.argv[2]
+destination_type = sys.argv[3]
+
+send_lib.check_destination_type_allowed(destination_type, send_lib.DESTINATION_TYPE_URL)
+send_lib.check_destination_format(destination, destination_type)
+
+auth = send_lib.get_auth_credentials(SERVICE_NAME, destination)
+
+send_lib.create_lock(SERVICE_NAME, destination)
+gen_folder = send_lib.get_gen_folder(facility, SERVICE_NAME)
+with send_lib.prepare_temporary_directory() as temp_dir:
+	send_lib.copy_files_to_directory(gen_folder, temp_dir)
+
+	with tarfile.open(temp_dir + '/perun_data.tar.gz', 'w:gz', format=tarfile.GNU_FORMAT) as archive:
+		archive.add(temp_dir, arcname='.')
+
+	with open(temp_dir + '/perun_data.tar.gz', 'rb') as compressed:
+		headers = {'Content-type': 'application/tar+gzip'}
+		response = requests.put(destination, headers=headers, data=compressed, auth=auth, timeout=TIMEOUT)
+		if not response.ok:
+			send_lib.die_with_error(
+				"Request ended with error code: " + str(response.status_code))