Add initial files

.gitignore

@@ -0,0 +1,13 @@
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# vscode
+.vscode/
+
+# custom
+
+config.yml
+venv/

README.md

@@ -0,0 +1 @@
+Warning: YAML files must have a `.yaml` extention. `.yml` does **not** work.

biscd/1

@@ -0,0 +1 @@
+Docker version 18.09.1, build 4c52b90

biscd/biscd/__init__.py

@@ -0,0 +1,38 @@
+from os import environ
+from pathlib import Path
+import confuse
+from flask import Flask
+from flask_bootstrap import Bootstrap
+
+app = Flask(__name__)
+Bootstrap(app)
+
+def read_config():
+    config_template = {
+        'config' : {
+            'flask_development_server' : {
+                'flask_env' : confuse.OneOf(['development', 'production']),
+                'server_host' : confuse.String(pattern="^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"),
+                'port' : confuse.Integer()
+            },
+            'logging' : {
+                'infofile' : confuse.Filename(),
+                'errorfile' : confuse.Filename()
+            },
+            'message' : confuse.String()
+        }
+    }
+    conf = confuse.Configuration('biscd', __name__)
+    valid = conf.get(config_template)
+    print('valid', valid)
+    print('dict', valid.__dict__)
+    print('message is', valid.config.message)
+    if False:#not conf.has_section('MISC'):
+        raise ValueError("Failed to open any configfile containing a MISC section. \
+            Please make sure the script dir contains a config.yml, \
+            or /etc/{__name__}.conf exists")
+    return conf['config']
+
+config = read_config()
+
+from biscd import routes, errors

biscd/biscd/config_default.yaml

@@ -0,0 +1,13 @@
+config:
+  flask_development_server:
+    # Options: production, development
+    flask_env: development
+    # Options: 127.0.0.1 , 0.0.0.0
+    server_host: 0.0.0.0
+    port: 5000
+
+  logging:
+    infofile: info.log
+    errorfile: error.log
+
+  message: Hoi

biscd/biscd/errors.py

@@ -0,0 +1,24 @@
+from flask import render_template, request
+from biscd import app, config, utils
+
+@app.errorhandler(400)
+def bad_request(error):
+    return render_template('400.html', title='400', error=error), 400
+
+@app.errorhandler(404)
+def not_found_error():
+    return render_template('404.html', title='404'), 404
+
+@app.errorhandler(500)
+def internal_error(error):
+    write_error(error)
+    return render_template('500.html', title='500'), 500
+
+def write_error(error):
+    timestamp = utils.get_timestamp()
+    remote = utils.get_remote_addr()
+    logline = f"{remote} {timestamp} {str(error)} {error.__dict__} \n"
+
+    logfile = config.get('LOGGING', "errorfile")
+    with open(logfile, 'a') as logfile:
+        logfile.write(logline)

biscd/biscd/routes.py

@@ -0,0 +1,8 @@
+from flask import render_template, abort
+from biscd import app, config, utils
+
+@app.route('/', methods=['GET'])
+@app.route('/index', methods=['GET'])
+def index():
+    return render_template('index.html', message=config['message'].get())
+

biscd/biscd/templates/400.html

@@ -0,0 +1,10 @@
+{% extends "base.html" %}
+
+{% block app_content %}
+    <h1>404 Page Not Found</h1>
+    <p><a href="{{ url_for('index') }}">Back</a></p>
+
+    <h1>400 Bad Request</h1>
+    <h2>Error Message:</h2>
+    {{ error }}
+{% endblock %}

biscd/biscd/templates/404.html

@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+
+{% block app_content %}
+    <h1>404 Page Not Found</h1>
+    <p><a href="{{ url_for('index') }}">Back</a></p>
+{% endblock %}

biscd/biscd/templates/500.html

@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+
+{% block app_content %}
+    <h1>500 Internal Server Error</h1>
+    <p><a href="{{ url_for('index') }}">Back</a></p>
+{% endblock %}

biscd/biscd/templates/base.html

@@ -0,0 +1,28 @@
+{% extends 'bootstrap/base.html' %}
+
+{% block styles %}
+{{ super() }}
+{% endblock %}
+
+{% block title %}
+{% if title %}{{ title }} - Biscd 🍪{% else %}Biscd 🍪{% endif %}
+{% endblock %}
+
+{% block content %}
+<div class="container">
+    {% with messages = get_flashed_messages() %}
+        {% if messages %}
+            {% for message in messages %}
+                <div class="alert alert-info" role="alert">{{ message }}</div>
+            {% endfor %}
+        {% endif %}
+    {% endwith %}
+
+    {# application content needs to be provided in the app_content block #}
+    {% block app_content %}{% endblock %}
+</div>
+{% endblock %}
+
+{% block scripts %}
+{{ super() }}
+{% endblock %}

biscd/biscd/templates/index.html

@@ -0,0 +1,6 @@
+{% extends "base.html" %}
+
+{% block app_content %}
+    <h1>Biscd, what else would you like with your Gitea?</h1>
+    <h2>{{ message }}</h2>
+{% endblock %}

biscd/example.yaml

@@ -0,0 +1,13 @@
+config:
+  flask_development_server:
+    # Options: production, development
+    flask_env: production
+    # Options: 127.0.0.1 , 0.0.0.0
+    server_host: 127.0.0.1
+    port: 5000
+
+  logging:
+    infofile: info.log
+    errorfile: error.log
+  
+  message: Hoi

biscd/projects.yaml

@@ -0,0 +1,9 @@
+projects:
+  - The Hunt:
+      url: thehunt
+      git_repo: https://git.sciuro.org/Burathar/The-Hunt
+      branch: master
+      secret: thisissecret
+      requirements-file: requirements.txt
+      tests: tests.py
+      

biscd/requirements.txt

@@ -0,0 +1,15 @@
+aniso8601==8.1.0
+certifi==2020.12.5
+chardet==4.0.0
+click==7.1.2
+Flask==1.1.2
+Flask-RESTful==0.3.8
+idna==2.10
+itsdangerous==1.1.0
+Jinja2==2.11.2
+MarkupSafe==1.1.1
+pytz==2020.5
+requests==2.25.1
+six==1.15.0
+urllib3==1.26.3
+Werkzeug==1.0.1

biscd/run.sh

@@ -0,0 +1,22 @@
+#! /bin/bash
+
+script_dir="$(dirname $(readlink -f $0))"
+app_name='biscd'
+source "$script_dir/venv/bin/activate"
+
+get_conf () {
+    echo `grep -s "$1" /etc/$app_name.yml "$script_dir/biscd/config_default.yaml" | cut -d ':' -f 3 | head -n 1`
+}
+
+enviroment=`get_conf 'flask_env'`
+host=`get_conf 'server_host'`
+port=`get_conf 'port'`
+
+export FLASK_APP="$script_dir/runserver.py"
+export FLASK_ENV="$enviroment"
+export APP_NAME="$app_name"
+
+flask run --host "$host" --port "$port"
+
+deactivate
+exit 0

biscd/runserver.py

@@ -0,0 +1 @@
+from biscd import app

install.sh

@@ -0,0 +1,211 @@
+#! /bin/bash
+
+script_dir="$(dirname $(readlink -f $0))"
+
+install_dir_parent="/opt"
+default_app_name='biscd'
+start_script='run.sh'
+
+# exit when any command fails
+set -e
+
+# Make sure running as root
+if [ `id -u` -ne 0 ]; then
+   echo 'Please run as root'
+   exit 1
+fi
+
+function help () {
+   echo "Usage: $0 [OPTIONS] <application name>"
+   echo ' -w     install this application as WSGI application. Overrules -d'
+   echo ' -d     install this application to be run using the flask development webserver.'
+   echo ' -u     update: ignore existing app/user/group warning (to update/overwrite already installed app).'
+   echo ' -p     port: On which port the internal webserver runs. Default = 5000. Ignored in WSGI mode'
+   echo ' -h     display this output.'
+   echo ''
+   echo "This script installs $default_app_name, by downloading dependancies when not present,"
+   echo 'creating a python virtualenv, unpacking any necessary files and creates a systemd integration if the user wishes so.'
+   echo ''
+   echo 'For more info visit https://git.sciuro.org/Burathar/biscd'
+   exit 1
+}
+
+function echo_header () {
+   echo -e "\e[33m== $1 ==\e[0;m"
+}
+
+unset use_wsgi
+while getopts ':uhdwp:' opt ; do
+   case "$opt" in
+      u) ignore_name='true';;
+      d) use_wsgi='false';;
+      w) use_wsgi='true';;
+      p) internal_port="${OPTARG}";;
+      h) help ;;
+      :)
+         echo "$0: Must supply an argument to -$OPTARG." >&2
+         exit 1
+         ;;
+      ?)
+         echo "Invalid option: -${OPTARG}."
+         exit 2
+         ;;
+   esac
+done
+arg_name=${@:$OPTIND:1}
+
+if [ -n "$arg_name" ]; then
+   [[ $arg_name =~ ^[a-zA-Z0-9][a-zA-Z0-9_-]*$ ]] || { echo "'$arg_name' is not allowed as an app name. Please only use letters, numbers, underscores(_) and dashes(-). The first character must be a letter or number." && exit 1; }
+   if [ "$ignore_name" != 'true' ]; then
+      compgen -G "/etc/systemd/system/$arg_name.*" >/dev/null && echo "An app called '$arg_name' already exists. Please choose another name, or run with -u to update existing app with this name." && exit 1
+      id -u "$arg_name" >/dev/null 2>&1 && echo "A user called '$arg_name' already exists. Please choose another name." && exit 1
+      egrep "^$arg_name" /etc/group >/dev/null && echo "A group called '$arg_name' already exists. Please choose another name." && exit 1
+   fi
+   app_name="$arg_name"
+else
+   app_name="$default_app_name"
+fi
+
+install_dir="$install_dir_parent/$app_name"
+logging_dir="/var/log/$app_name"
+
+if [ -z "$use_wsgi" ]; then
+   echo "Should this application be installed as WSGI app (recommended), or as local daemon using flask's built-in webserver, with apache as a proxy?"
+   read -p "Use WSGI? (Y/n) " use_wsgi
+   [ -z "$use_wsgi" ] && use_wsgi='y' # if no input, assume yes
+   case ${use_wsgi:0:1} in
+      y|Y|1 )
+         use_wsgi='true';;
+      * )
+         use_wsgi='false';;
+   esac
+fi
+
+if [ "$use_wsgi" != 'true' ]; then
+   echo "Do you want to daemonize this application using systemd? (you'll have to start it manually every login session if you choose no)"
+   read -p "Use Systemd? (Y/n) " use_systemd
+   [ -z "$use_systemd" ] && use_systemd='y' # if no input, assume yes
+   case ${use_systemd:0:1} in
+      y|Y|1 )
+         use_systemd='true';;
+      * )
+         use_systemd='false';;
+   esac
+
+   if [ -z "$internal_port" ]; then
+      echo "Please specify an internal port for the built-in webserver. This port will only be used locally."
+      read -p "Port [5000]" internal_port
+      [ -z "$internal_port" ] && internal_port=5000
+   fi
+fi
+
+echo_header "Check if Apache2 is installed"
+which apache2 >/dev/null 2>&1 && echo "Apache OK" || { echo "Apache2 doesn't seem te be installed. Please fix this before installing this application." && exit 1; }
+
+if [ "$use_wsgi" = 'true' ]; then
+   echo_header "Make sure mod-wsgi-py3 is installed"
+   apt-get install -y libapache2-mod-wsgi-py3
+fi
+
+echo_header "Copy over application files"
+mkdir -vp "$install_dir"
+cp -rv "$script_dir/$default_app_name/." "$install_dir"
+sed -i "s+^app_name=.*+app_name='${flask_app_name}'+g" "$install_dir/run.sh"
+
+cp -v "$script_dir/$default_app_name/config_example.yml" "$install_dir/config.yml"
+sed -i "s/^port =.*/port = $internal_port/g" "$install_dir/config.yml"
+sed -i "s+^logfile :.*+logfile : $logging_dir/reports.log+g" "$install_dir/config.yml"
+sed -i "s+^errorfile :.*+errorfile : $logging_dir/error.log+g" "$install_dir/config.yml"
+
+cp -v "$script_dir/bin/uninstall.sh" "$install_dir"
+sed -i "s+^app_name=.*+app_name='${app_name}'+g" "$install_dir/uninstall.sh"
+sed -i "s+^install_dir=.*+install_dir='${install_dir}'+g" "$install_dir/uninstall.sh"
+
+cp -v "$script_dir/version" "$install_dir"
+cp -v "$script_dir/README.md" "$install_dir"
+
+
+echo_header "Copy over and enable apache vhost"
+if [ "$use_wsgi" = 'true' ]; then
+   cp -v "$script_dir/bin/$default_app_name-wsgi.conf" "/etc/apache2/sites-available/$app_name.conf"
+   sed -i "s/APPNAME/$app_name/g" "/etc/apache2/sites-available/$app_name.conf"
+   sed -i "s/USER/$app_name/g" "/etc/apache2/sites-available/$app_name.conf"
+   sed -i "s/GROUP/$app_name/g" "/etc/apache2/sites-available/$app_name.conf"
+
+   ln -sfv "/etc/apache2/sites-available/$app_name.conf" "/etc/apache2/sites-enabled/$app_name.conf"
+
+   echo_header "Installing wsgi-script"
+   mkdir -vp "/var/www/wsgi-scripts"
+   cp -v "$script_dir/bin/$default_app_name.wsgi" "/var/www/wsgi-scripts/app_name.wsgi"
+   sed -i "s+INSTALLDIR+$install_dir+g" "/var/www/wsgi-scripts/$app_name.wsgi"
+   sed -i "s/APPNAME/$flask_app_name/g" "/var/www/wsgi-scripts/$app_name.wsgi"
+else
+   cp -v "$script_dir/bin/$default_app_name-proxy.conf" "/etc/apache2/sites-available/$app_name.conf"
+   sed -i "s/PORT/$internal_port/g" "/etc/apache2/sites-available/$app_name.conf"
+   
+   ln -sfv "/etc/apache2/sites-available/$app_name.conf" "/etc/apache2/sites-enabled/$app_name.conf"
+fi
+
+
+echo_header "Create $app_name user"
+adduser --debug --system --home "$install_dir" --shell "/usr/sbin/nologin" --group --gecos "CSP violation report application" -q "$app_name"
+usermod -a -G "$app_name" root # Add root to group to prevent warnings when editing config file
+
+
+echo_header "Create logging directory"
+mkdir -p "$logging_dir"
+chown -v --from=root:root root:"$app_name" "$logging_dir"
+chmod -v 770 "$logging_dir"
+touch "$logging_dir/reports.log"
+chown -v --from=root:root "$app_name":"$app_name" "$logging_dir/reports.log"
+
+
+echo_header "Install Logrotate config"
+cp -v "$script_dir/bin/logrotate-conf" "/etc/logrotate.d/$app_name"
+sed -i "s/log-dir/$logging_dir/g" "/etc/logrotate.d/$app_name"
+
+
+echo_header "Make sure python3 and virtualenv are installed"
+python3 --version || apt-get install -y python3
+
+# Testing to check if venv is installed does not work, venv command can respond to a versioncheck, and still not have neccesary libraries present.
+apt-get install -y python3-venv
+
+
+echo_header "Create virualenv"
+[ -f "$install_dir/venv/bin/activate" ] || python3 -m venv "$install_dir/venv"
+source "$install_dir/venv/bin/activate"
+pip install setuptools wheel
+pip install -r "$script_dir/$default_app_name/requirements.txt"
+
+python_version=`ls "$install_dir/venv/lib" | grep python3 | head -1`
+echo "$install_dir/" > "$install_dir/venv/lib/$python_version/site-packages/$flask_app_name.pth"
+[ "$use_wsgi" = 'true' ] && sed -i "s/PYTHON_VERSION/$python_version/g" "/var/www/wsgi-scripts/$default_app_name.wsgi"
+
+
+echo_header "Give $app_name user ownership of application dir"
+chown --recursive "$app_name":"$app_name" "$install_dir"
+chown -v root:"$app_name" "$install_dir"
+chmod -v 775 "$install_dir"
+chmod -v 754 "$install_dir/run.sh" "$install_dir/runserver.py"
+chmod -v 744 "$install_dir/uninstall.sh"
+#chmod -v 660 "$install_dir/config.yml"
+
+
+if [ "$use_wsgi" != 'true' ] && [ "$use_systemd" = 'true' ]; then
+   echo_header "Enable as systemd service"
+   cp "$script_dir/bin/$default_app_name.service" "/etc/systemd/system/$app_name.service"
+   sed -i "s+^ExecStart=.*+ExecStart=${install_dir}/${start_script}+g" "/etc/systemd/system/$app_name.service"
+   sed -i "s+^User=.*+User=${app_name}+g" "/etc/systemd/system/$app_name.service"
+   systemctl daemon-reload
+   systemctl enable "$app_name.service"
+   systemctl restart "$app_name.service"
+   systemctl status "$app_name.service"
+fi
+
+
+echo_header "$app_name is installed!"
+
+echo "If everything works, it is safe to remove this installation directory"
+echo -e "\e[1;35mNow please review/edit /etc/apache2/sites-enabled/$app_name.conf, and then run 'systemctl restart apache2.service'\e[0m"
+exit 0

installation-files/biscd-proxy.conf

@@ -0,0 +1,18 @@
+LoadModule proxy_module /usr/lib/apache2/modules/mod_proxy.so
+LoadModule proxy_http_module /usr/lib/apache2/modules/mod_proxy_http.so
+
+<VirtualHost *:80>
+   #ServerAdmin webmaster@localhost
+   #ServerName www.example.com
+   #ServerAlias example.com
+
+   ErrorLog ${APACHE_LOG_DIR}/error.log
+   CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+   # Reverse Proxy definition
+   ProxyPass / http://127.0.0.1:PORT/
+   ProxyPassReverse / http://127.0.0.1:PORT/
+
+</VirtualHost>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

installation-files/biscd-wsgi.conf

@@ -0,0 +1,31 @@
+LoadModule wsgi_module /usr/lib/apache2/modules/mod_wsgi.so
+
+<VirtualHost *:80>
+   #ServerAdmin webmaster@localhost
+   #ServerName www.example.com
+   #ServerAlias example.com
+
+   ErrorLog ${APACHE_LOG_DIR}/error.log
+   CustomLog ${APACHE_LOG_DIR}/access.log combined
+
+   LogLevel info
+
+   WSGIDaemonProcess APPNAME user=USER group=GROUP threads=5
+   WSGIScriptAlias / /var/www/wsgi-scripts/APPNAME.wsgi
+
+   <Directory /var/www/wsgi-scripts>
+      WSGIProcessGroup APPNAME
+      WSGIApplicationGroup %{GLOBAL}
+
+      <IfVersion < 2.4>
+         Order allow,deny
+         Allow from all
+      </IfVersion>
+      <IfVersion >= 2.4>
+         Require all granted
+      </IfVersion>
+   </Directory>
+
+</VirtualHost>
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet

installation-files/biscd.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Starts biscd, a light-weight ci-cd tool
+# Require syslog.socket to make sure this doesn't start to soon after boot
+Requires=syslog.socket
+Documentation=https://git.sciuro.org/Burathar/biscd
+StartLimitIntervalSec=10
+StartLimitBurst=3
+
+[Service]
+Type=simple
+User=biscd
+ExecStart=/opt/biscd/run.sh
+Restart=always
+RestartSec=1
+
+[Install]
+WantedBy=default.target
+

installation-files/biscd.wsgi

@@ -0,0 +1,7 @@
+import site
+site.addsitedir('INSTALLDIR/venv/lib/PYTHON_VERSION/site-packages') 
+#virtualenv = 'INSTALLDIR/venv/bin/activate'
+#with open(virtualenv) as file_:
+#    exec(file_.read(), dict(__file__=virtualenv))
+
+from APPNAME import app as application

installation-files/logrotate-conf

@@ -0,0 +1,13 @@
+log-dir/info.log {
+    size 100k
+    rotate 2
+    compress
+    missingok
+}
+
+log-dir/error.log {
+    size 100k
+    rotate 2
+    compress
+    missingok
+}

installation-files/uninstall.sh

@@ -0,0 +1,103 @@
+#! /bin/bash
+
+script_dir="$(dirname $(readlink -f $0))"
+
+app_name='biscd'
+install_dir="/opt/$app_name"
+logging_dir="/var/log/$app_name"
+
+systemd_service=`systemctl list-unit-files | cut -d ' ' -f 1 | grep -swcF "$app_name.service"` > 0
+
+# exit when any command fails
+set -e
+
+function help () {
+   echo "Usage: $0 [OPTIONS]"
+   echo " -f     force uninstallation: Don't stop when something fails"
+   echo ' -l     remove logfiles and logdirectory as well'
+   echo ' -h     display this output.'
+   echo ''
+   echo "This script uninstalls $app_name. All application files will be removed,"
+   echo 'as will the logrotate and apache config(section).'
+   echo 'Apache will be reloaded in this process.'
+   echo ''
+   echo 'For more info visit https://git.sciuro.org/Burathar/biscd'
+   exit 1
+}
+
+function echo_header () {
+   echo -e "\e[33m== $1 ==\e[0;m"
+}
+
+while getopts 'flh' opt ; do
+   case "$opt" in
+      f) set +e; force='true';;
+      l) delete_logs='true';;
+      h) help ;;
+   esac
+done
+
+# Make sure running as root
+if [ `id -u` -ne 0 ]; then
+   echo 'Please run as root'
+   exit 1
+fi
+
+if [ "$force" != 'true' ]; then
+   read -p "Are you sure you want to delete $app_name? Apache will be reloaded during this process (y/N) " confirm_delete
+   [ -z "$confirm_delete" ] && confirm_delete='no' # if no input, assume no
+   case ${confirm_delete:0:1} in
+      n|N|0 )
+         exit 0;;
+   esac
+fi
+
+if [ "$delete_logs" != 'true' ]; then
+   read -p "Do you wish to delete any $app_name log files as well? (y/N) " delete_logs
+   [ -z "$delete_logs" ] && delete_logs='no' # if no input, assume no
+   case ${delete_logs:0:1} in
+      n|N|0 )
+         delete_logs='false';;
+      *)
+         delete_logs='true';;
+   esac
+fi
+
+echo_header "Removing Apache virtualhost"
+rm -v "/etc/apache2/sites-enabled/$app_name.conf"
+systemctl reload apache2.service
+
+
+if [ "$systemd_service" -eq 1 ]; then
+   echo_header "Removing Systemd service"
+   systemctl stop "$app_name.service"
+   sleep 1s
+   systemctl disable "$app_name.service"
+   rm "/etc/systemd/system/$app_name.service"
+   systemctl daemon-reload
+fi
+
+
+echo_header "Removing $app_name user and group=="
+deluser -q "$app_name"
+delgroup -q "$app_name" && echo "group $app_name is removed, ignore previous warning" || true
+
+echo_header "Removing application files"
+rm -v "/etc/logrotate.d/$app_name"
+rm -v "/etc/apache2/sites-available/$app_name.conf"
+rm -v "/var/www/wsgi-scripts/$app_name.wsgi" || true
+rm -vd "/var/www/wsgi-scripts/" || true
+rm -rf "$install_dir" && echo "removed '$install_dir'"
+
+if [ "$delete_logs" = 'true' ]; then
+echo_header "Removing logfiles"
+   rm -vrf "$logging_dir"
+fi
+
+echo_header "Remaining artifacts(feel free to remove these manually)"
+dpkg -l | grep libapache2-mod-wsgi-py3 | awk '{print $2}'
+echo python3-venv
+
+echo_header "$app_name is uninstalled!"
+
+exit 0