Packaging v2 (#399)

* Convert to v2 (#383)

* convert script

* v2

* Auto-update README

* Update documentation cc @panomaki

* Bump ruby version from 3.0.6 to 3.2.2

* v2

* Auto-update README

* woops

* v2

* v2

* s/final_path/install_dir

* Update PRE_INSTALL.md

* too soon...

* v2

* v2

* v2

* Update tests.toml

* Update doc/PRE_INSTALL.md

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>

* Update doc/PRE_INSTALL_fr.md

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>

* Update tests.toml

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>

* Update tests.toml

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>

* Update doc/PRE_INSTALL.md

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>

* Update doc/PRE_INSTALL_fr.md

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>

* Apply suggestions

* Update manifest.toml

* Auto-update README

* Update manifest.toml

* Update scripts

* Auto-update README

* Remove SSO integration

* Use new mail mechanism

* cleaning

* Update manifest.toml

---------

Co-authored-by: yunohost-bot <yunohost@yunohost.org>
Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>
Co-authored-by: Éric Gaspar <46165813+ericgaspar@users.noreply.github.com>

* Bump ynh version

* Auto-update README

* Update manifest.toml

* Update doc/PRE_INSTALL.md

Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>

* Add service setting for app shell

* Use logrotate

* smtp config follow-up

Sidekiq service can be properly hardened since it doesn't use sendmail anymore o/

* Fix mkdir failing if /var/log/mastodon already exists

* upgrade: Rename database to match packaging v2

* Bump 4.1.6~ynh2

* Auto-update README

* Disable install.private test

* remove full_domain

* 4.1.7 (#403)

* 4.1.7

* Auto-update README

* Update manifest.toml

* Auto-update README

---------

Co-authored-by: yunohost-bot <yunohost@yunohost.org>
Co-authored-by: Tagada <36127788+Tagadda@users.noreply.github.com>

* fix

* fix

* Don't swap when running on a LXC

* Hardcore the needed RAM

* Auto-update README

* Auto-update README

* Update manifest.toml

* Auto-update README

* Update _common.sh

* Auto-update README

* cleaning

* cleaning

* Auto-update README

* Auto-update README

* Update tests.toml

* Fix restore

* Fix manifest.toml

* Auto-update README

* Auto-update README

* add `tootctl self-destruct` info

* format

* Auto-update README

* lol autotranslate

* break line

* break lines

* translation from french

* update version

* Auto-update README

* Update manifest.toml

---------

Co-authored-by: yunohost-bot <yunohost@yunohost.org>
Co-authored-by: Alexandre Aubin <alex.aubin@mailoo.org>
Co-authored-by: Éric Gaspar <46165813+ericgaspar@users.noreply.github.com>
Co-authored-by: ericgaspar <junk.eg@free.fr>
Co-authored-by: OniriCorpe <oniricorpe@disroot.org>

1 files changed, 69 insertions, 221 deletions

diff --git a/scripts/upgrade b/scripts/upgrade
index e041939..aaafcc0 100644
--- a/scripts/upgrade
+++ b/scripts/upgrade
@@ -11,231 +11,82 @@ source ynh_install_ruby__2
 source ynh_add_swap
 source /usr/share/yunohost/helpers
 
-#=================================================
-# LOAD SETTINGS
-#=================================================
-ynh_script_progression --message="Loading installation settings..." --weight=1
-
-app=$YNH_APP_INSTANCE_NAME
-
-domain=$(ynh_app_setting_get --app=$app --key=domain)
-path_url=$(ynh_app_setting_get --app=$app --key=path)
-language=$(ynh_app_setting_get --app=$app --key=language)
-admin=$(ynh_app_setting_get --app=$app --key=admin)
-final_path=$(ynh_app_setting_get --app=$app --key=final_path)
-redis_namespace=$(ynh_app_setting_get --app=$app --key=db_name)
-db_name=$(ynh_app_setting_get --app=$app --key=db_name)
-db_user=$(ynh_app_setting_get --app=$app --key=db_user)
-db_pwd=$(ynh_app_setting_get --app=$app --key=psqlpwd)
-admin_mail=$(ynh_user_get_info --username=$admin --key='mail')
-port_web=$(ynh_app_setting_get --app=$app --key=port_web)
-port_stream=$(ynh_app_setting_get --app=$app --key=port_stream)
-
-secret_key_base=$(ynh_app_setting_get --app=$app --key=secret_key_base)
-otp_secret=$(ynh_app_setting_get --app=$app --key=otp_secret)
-vapid_private_key=$(ynh_app_setting_get --app=$app --key=vapid_private_key)
-vapid_public_key=$(ynh_app_setting_get --app=$app --key=vapid_public_key)
-
-config="$final_path/live/.env.production"
-
-#=================================================
-# CHECK VERSION
-#=================================================
-ynh_script_progression --message="Checking version..." --weight=1
-
 upgrade_type=$(ynh_check_app_version_changed)
-
-#=================================================
-# BACKUP BEFORE UPGRADE THEN ACTIVE TRAP
-#=================================================
-ynh_script_progression --message="Backing up the app before upgrading (may take a while)..." --weight=1
-
-# Backup the current version of the app
-ynh_backup_before_upgrade
-ynh_clean_setup () {
-	# Restore it if the upgrade fails
-	ynh_restore_upgradebackup
-}
-# Exit if an error occurs during the execution of the script
-ynh_abort_if_errors
+config="$install_dir/live/.env.production"
 
 #=================================================
 # STANDARD UPGRADE STEPS
 #=================================================
-# STOP SYSTEMD SERVICE
-#=================================================
-ynh_script_progression --message="Stopping a systemd service..." --weight=1
-
-ynh_systemd_action --service_name=${app}-web --action="stop" --log_path=systemd
-ynh_systemd_action --service_name=${app}-sidekiq --action="stop" --log_path=systemd
-ynh_systemd_action --service_name=${app}-streaming --action="stop" --log_path=systemd
-
-#=================================================
 # ENSURE DOWNWARD COMPATIBILITY
 #=================================================
 ynh_script_progression --message="Ensuring downward compatibility..." --weight=1
 
-# Cleaning legacy permissions
-if ynh_legacy_permissions_exists; then
-	ynh_legacy_permissions_delete_all
-
-	ynh_app_setting_delete --app=$app --key=is_public
-fi
-
-# Create a permission if needed
-if ! ynh_permission_exists --permission="api"; then
-	ynh_permission_create --permission="api" --url="/api" --allowed="visitors" --auth_header="false" --show_tile="false" --protected="true"
-fi
-
-# If port_web doesn't exist, create it, needed for old install
-if [[ -z "$port_web" ]]; then
-	port_web=3000
-	ynh_app_setting_set --app=$app --key=port_web --value=$port_web
-fi
-
-# If port_web doesn't exist, create it, needed for old install
-if [[ -z "$port_stream" ]]; then
-	port_stream=4000
-	ynh_app_setting_set --app=$app --key=port_stream --value=$port_stream
-fi
-
-# If db_user doesn't exist, create it, needed for old install
-if [[ -z "$db_user" ]]; then
-	db_user=$(ynh_sanitize_dbid --db_name=$app)
-	ynh_app_setting_set --app=$app --key=db_user --value=$db_user
-fi
-
-# If db_pwd doesn't exist, create it, needed for old install
-if [[ -z "$db_pwd" ]]; then
-	db_pwd=$(ynh_string_random)
-	ynh_app_setting_set --app=$app --key=db_pwd --value=$db_pwd
-	ynh_psql_test_if_first_run
-	sudo --login --user=postgres psql -c"ALTER user $app WITH PASSWORD '$db_pwd'" postgres
-	ynh_replace_string --match_string="DB_PASS=" --replace_string="DB_PASS=${db_pwd}" --target_file="$config"
-fi
-
-# Remove paperclip_secret
-ynh_app_setting_delete --app=$app --key=paperclip_secret
-
-# If secret_key_base doesn't exist, retrieve it or create it
-if [[ -z "$secret_key_base" ]]; then
-	secret_key_base=$(grep -oP "SECRET_KEY_BASE=\K\w+" $config)
-	if [[ -z "$secret_key_base" ]]; then
-		secret_key_base=$(ynh_string_random --length=128)
-	fi
-	ynh_app_setting_set --app=$app --key=secret_key_base --value="$secret_key_base"
-fi
-
-# If otp_secret doesn't exist, retrieve it or create it
-if [[ -z "$otp_secret" ]]; then
-	otp_secret=$(grep -oP "OTP_SECRET=\K\w+" $config)
-	if [[ -z "$otp_secret" ]]; then
-		otp_secret=$(ynh_string_random --length=128)
-	fi
-	ynh_app_setting_set --app=$app --key=otp_secret --value="$otp_secret"
-fi
-
-# If vapid_private_key doesn't exist, retrieve it or create it
-if [[ -z "$vapid_private_key" ]]; then
-	vapid_private_key=$(grep -oP "VAPID_PRIVATE_KEY=\K.+" $config)
-	vapid_public_key=$(grep -oP "VAPID_PUBLIC_KEY=\K.+" $config)
-	ynh_app_setting_set "$app" vapid_private_key "$vapid_private_key"
-	ynh_app_setting_set "$app" vapid_public_key "$vapid_public_key"
-fi
-
-# If redis_namespace doesn't exist, create it
-if [[ -z "$redis_namespace" ]]; then
-	redis_namespace=${app}_production
-	ynh_app_setting_set --app=$app --key=redis_namespace --value=$redis_namespace
+# Set `service` settings to support `yunohost app shell` command
+if [[ -z "${service:-}" ]]; then
+	service="$app-web.service"
+	ynh_app_setting_set --app="$app" --key=service --value="$service"
 fi
 
 #Remove previous added repository
 ynh_remove_extra_repo
 
 #=================================================
-# CREATE DEDICATED USER
+# STOP SYSTEMD SERVICE
 #=================================================
-ynh_script_progression --message="Making sure dedicated system user exists..." --weight=1
+ynh_script_progression --message="Stopping a systemd service..." --weight=1
 
-# Create a dedicated user (if not existing)
-ynh_system_user_create --username=$app --home_dir="$final_path"
+ynh_systemd_action --service_name=${app}-web --action="stop" --log_path=/var/log/$app/$app-web.log --line_match="Goodbye"
+ynh_systemd_action --service_name=${app}-sidekiq --action="stop" --log_path=/var/log/$app/$app-sidekiq.log --line_match="Bye"
+ynh_systemd_action --service_name=${app}-streaming --action="stop" --log_path=/var/log/$app/$app-streaming.log --line_match="exiting"
 
 #=================================================
-# DOWNLOAD, CHECK AND UNPACK SOURCE
+# ADD SWAP IF NEEDED
 #=================================================
+ynh_script_progression --message="Adding swap if needed..." --weight=1
 
-if [ "$upgrade_type" == "UPGRADE_APP" ]
-then
-	ynh_script_progression --message="Upgrading source files..." --weight=1
-
-	# Download Mastodon
-	tmpdir="$(mktemp -d)"
-
-	mkdir $tmpdir/system
-	if [ -d "$final_path/live/public/system" ]; then
-		mv --verbose --no-target-directory --backup=numbered "$final_path/live/public/system" "$final_path/system.tmp"
-	fi
-	rsync -a "$config" "$tmpdir/."
-	ynh_secure_remove --file="$final_path/live"
-	
-	ynh_setup_source --dest_dir="$final_path/live"
-
-	if [ -d "$final_path/system.tmp" ]; then
-		mv --verbose --no-target-directory "$final_path/system.tmp" "$final_path/live/public/system"
-	fi
-	rsync -a "$tmpdir/.env.production" "$final_path/live/."
-	ynh_secure_remove --file="$tmpdir"
+total_memory=$(ynh_get_ram --total)
+swap_needed=0
 
-	# Clean files which are not needed anymore
-	ynh_secure_remove --file="$final_path/live/config/initializers/timeout.rb"
+if [ $total_memory -lt $memory_needed ]; then
+	# Need a minimum of 8Go of memory
+	swap_needed=$(($memory_needed - $total_memory))
 fi
 
-chmod 750 "$final_path"
-chmod -R o-rwx "$final_path"
-chown -R $app:www-data "$final_path"
+ynh_script_progression --message="Adding $swap_needed Mo to swap..."
+ynh_add_swap --size=$swap_needed
 
 #=================================================
 # UPGRADE DEPENDENCIES
 #=================================================
-ynh_script_progression --message="Upgrading dependencies..." --weight=1
+ynh_script_progression --message="Upgrading Ruby and NodeJS..." --weight=1
 
-ynh_exec_warn_less ynh_install_app_dependencies $pkg_dependencies $build_pkg_dependencies
 ynh_exec_warn_less ynh_install_ruby --ruby_version=$ruby_version
 ynh_exec_warn_less ynh_install_nodejs --nodejs_version=$nodejs_version
-ynh_exec_warn_less ynh_install_extra_app_dependencies --repo="deb https://dl.yarnpkg.com/debian/ stable main" --package="yarn" --key="https://dl.yarnpkg.com/debian/pubkey.gpg"
 
 #=================================================
-# NGINX CONFIGURATION
+# "REBUILD" THE APP (DEPLOY NEW SOURCES, RERUN NPM BUILD...)
 #=================================================
-ynh_script_progression --message="Upgrading NGINX web server configuration..." --weight=1
-
-# Create a dedicated NGINX config
-ynh_add_nginx_config
-
-#=================================================
-# SPECIFIC UPGRADE
-#=================================================
-# ADD SWAP IF NEEDED
+# DOWNLOAD, CHECK AND UNPACK SOURCE
 #=================================================
-ynh_script_progression --message="Adding swap if needed..." --weight=1
 
-total_memory=$(ynh_get_ram --total)
-swap_needed=0
+if [ "$upgrade_type" == "UPGRADE_APP" ]
+then
+	ynh_script_progression --message="Upgrading source files..." --weight=1
 
-if [ $total_memory -lt $memory_needed ]; then
-	# Need a minimum of 8Go of memory
-	swap_needed=$(($memory_needed - $total_memory))
-fi
+	# Download Mastodon
+	ynh_setup_source --dest_dir="$install_dir/live" --keep="public/system/"
 
-ynh_script_progression --message="Adding $swap_needed Mo to swap..."
-ynh_add_swap --size=$swap_needed
+	chmod 750 "$install_dir"
+	chmod -R o-rwx "$install_dir"
+	chown -R $app:www-data "$install_dir"
+fi
 
 #=================================================
-# BUILD APP
+# BUILD ASSETS
 #=================================================
-ynh_script_progression --message="Building app..." --weight=1
+ynh_script_progression --message="Building assets..." --weight=1
 
-pushd "$final_path/live"
+pushd "$install_dir/live"
 	ynh_use_ruby
 	ynh_gem update --system
 	ynh_gem install bundler --no-document
@@ -244,71 +95,68 @@ pushd "$final_path/live"
 	ynh_exec_as $app $ynh_ruby_load_path $ld_preload bin/bundle config set force_ruby_platform true --quiet
 	ynh_exec_as $app $ynh_ruby_load_path $ld_preload bin/bundle install -j$(getconf _NPROCESSORS_ONLN)
 	ynh_use_nodejs
-	ynh_exec_as $app $ynh_node_load_PATH yarn install --pure-lockfile
+	ynh_exec_as $app $ynh_node_load_PATH yarn install --pure-lockfile --production --network-timeout 600000
 	ynh_exec_warn_less ynh_exec_as $app RAILS_ENV=production $ynh_ruby_load_path $ld_preload bin/bundle exec rails assets:clean
 	ynh_exec_warn_less ynh_exec_as $app RAILS_ENV=production $ynh_ruby_load_path $ld_preload bin/bundle exec rails assets:precompile
-	ynh_exec_warn_less ynh_exec_as $app RAILS_ENV=production $ynh_ruby_load_path $ld_preload bin/bundle exec rails db:migrate
-	ynh_exec_warn_less ynh_exec_as $app RAILS_ENV=production $ynh_ruby_load_path $ld_preload bin/tootctl cache clear
 popd
 
-ynh_exec_warn_less ynh_install_app_dependencies $pkg_dependencies
-ynh_package_autoremove
-
 #=================================================
-# UPDATE A CONFIG FILE
+# REAPPLY SYSTEM CONFIGURATIONS
 #=================================================
-ynh_script_progression --message="Updating a config file..." --weight=1
-
-language="$(echo $language | head -c 2)"
+ynh_script_progression --message="Upgrading system configurations related to $app..." --weight=1
 
-ynh_add_config --template="../conf/.env.production.sample" --destination="$config"
-
-chmod 400 "$config"
-chown $app:$app "$config"
-
-#=================================================
-# SETUP SYSTEMD
-#=================================================
-ynh_script_progression --message="Upgrading systemd configuration..." --weight=1
+ynh_add_nginx_config
 
 # Create a dedicated systemd config
 ynh_add_systemd_config --service="$app-web" --template="mastodon-web.service"
+yunohost service add "$app-web" --description="$app web service"
+
 ynh_add_systemd_config --service="$app-sidekiq" --template="mastodon-sidekiq.service"
+yunohost service add "$app-sidekiq" --description="$app sidekiq service"
+
 ynh_add_systemd_config --service="$app-streaming" --template="mastodon-streaming.service"
+yunohost service add "$app-streaming" --description="$app streaming service"
 
-#=================================================
-# SETUP THE CRON FILE
-#=================================================
-ynh_script_progression --message="Setuping the cron file..." --weight=1
+ynh_add_config --template="cron" --destination="/etc/cron.d/$app"
 
-ynh_add_config --template="../conf/cron" --destination="/etc/cron.d/$app"
+# Use logrotate to manage app-specific logfile(s)
+mkdir -p /var/log/$app
+chown $app:$app /var/log/$app
+ynh_use_logrotate --non-append
 
 #=================================================
-# GENERIC FINALIZATION
+# RECONFIGURE THE APP (UPDATE CONF, APPLY MIGRATIONS...)
 #=================================================
-# INTEGRATE SERVICE IN YUNOHOST
+# UPDATE A CONFIG FILE
 #=================================================
-ynh_script_progression --message="Integrating service in YunoHost..." --weight=1
+ynh_script_progression --message="Updating a config file..." --weight=1
 
-yunohost service add "$app-web" --description="$app web service"
-yunohost service add "$app-sidekiq" --description="$app sidekiq service"
-yunohost service add "$app-streaming" --description="$app streaming service"
+language="$(echo $language | head -c 2)"
+
+ynh_add_config --template=".env.production.sample" --destination="$config"
+
+chmod 400 "$config"
+chown $app:$app "$config"
 
 #=================================================
-# START SYSTEMD SERVICE
+# APPLY MIGRATIONS
 #=================================================
-ynh_script_progression --message="Starting a systemd service..." --weight=1
+ynh_script_progression --message="Applying migrations..." --weight=1
 
-ynh_systemd_action --service_name=${app}-web --action="start" --log_path=systemd --line_match="Listening on"
-ynh_systemd_action --service_name=${app}-sidekiq --action="start" --log_path=systemd --line_match="Schedules Loaded"
-ynh_systemd_action --service_name=${app}-streaming --action="start" --log_path=systemd --line_match="Streaming API now listening"
+pushd "$install_dir/live"
+	ynh_use_ruby
+	ynh_exec_warn_less ynh_exec_as $app RAILS_ENV=production $ynh_ruby_load_path $ld_preload bin/bundle exec rails db:migrate
+	ynh_exec_warn_less ynh_exec_as $app RAILS_ENV=production $ynh_ruby_load_path $ld_preload bin/tootctl cache clear
+popd
 
 #=================================================
-# RELOAD NGINX
+# START SYSTEMD SERVICE
 #=================================================
-ynh_script_progression --message="Reloading NGINX web server..." --weight=1
+ynh_script_progression --message="Starting a systemd service..." --weight=1
 
-ynh_systemd_action --service_name=nginx --action=reload
+ynh_systemd_action --service_name=${app}-web --action="start" --log_path=/var/log/$app/$app-web.log --line_match="Listening on"
+ynh_systemd_action --service_name=${app}-sidekiq --action="start" --log_path=/var/log/$app/$app-sidekiq.log --line_match="Schedules Loaded"
+ynh_systemd_action --service_name=${app}-streaming --action="start" --log_path=/var/log/$app/$app-streaming.log --line_match="Streaming API now listening"
 
 #=================================================
 # END OF SCRIPT