diff --git a/Dockerfile b/Dockerfile index a8c6a3e..b75b366 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,31 +1,41 @@ +####################################### +#Dockerfile to build a ganetimgr image# +#Uses Deban packages instead of pip # +####################################### + +# We use wheezy as a base (for now) FROM debian:wheezy +MAINTAINER GRNET_NOC +ENV GANETIMGR_UPSTREAM_URL https://github.com/grnet/ganetimgr.git + +# First layer - only system packages, nothing from the stack +RUN apt-get update -q2 && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -q2 git procps apt-utils + +# Django and rest of python dependencies for the project +RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -q2 python-django python-redis python-mysqldb python-django-south python-django-registration python-paramiko python-simplejson python-daemon python-setproctitle python-pycurl python-recaptcha python-ipaddr python-bs4 python-requests python-markdown python-gevent -RUN apt-get update -q -RUN DEBIAN_FRONTEND=noninteractive apt-get install --quiet --yes git procps apt-utils -RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends --quiet --yes python-django python-redis python-mysqldb python-django-south python-django-registration python-paramiko python-simplejson python-daemon python-setproctitle python-pycurl python-recaptcha python-ipaddr python-bs4 python-requests python-markdown -RUN DEBIAN_FRONTEND=noninteractive apt-get install --quiet --yes gunicorn python-gevent beanstalkd nginx redis-server +# Daemon dependencies +RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -q2 gunicorn beanstalkd nginx redis-server +# This is a workaound for a project dependency that has no Debian package ADD python-django-markdown_0.6.1-1_all.deb / RUN dpkg -i /python-django-markdown_0.6.1-1_all.deb +# Can be removed when the commit that removes the dep is on master -WORKDIR /srv -RUN git clone --quiet https://github.com/grnet/ganetimgr.git +# Get the repository and switch context inside it +ENV GANETIMGR_INSTALLDIR=/srv/ganetimgr +RUN git clone --quiet $GANETIMGR_UPSTREAM_URL $GANETIMGR_INSTALLDIR +WORKDIR $GANETIMGR_INSTALLDIR -COPY settings.py ganetimgr/ganetimgr/settings.py +# Predifined Settings for use inside the container +COPY settings.py $GANETIMGR_INSTALLDIR/ganetimgr/settings.py +# Helper function to get the db connection info from envvars COPY dj_database_url.py ganetimgr/dj_database_url.py -COPY ganetimgr.nginx.conf /etc/nginx/nginx.conf -COPY ganetimgr.gunicorn.conf /etc/gunicorn/ganetimgr.conf -COPY beanstalkd.conf /etc/default/beanstalkd -#COPY run.sh / -#RUN cp ganetimgr/ganetimgr/settings.py.dist ganetimgr/ganetimgr/settings.py -RUN cp ganetimgr/templates/includes/analytics.html.dist ganetimgr/templates/includes/analytics.html -RUN ./ganetimgr/manage.py syncdb --noinput -#RUN echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', 'admin@example.com', DJANGO_ADMIN_PASS)" | python manage.py shell -RUN ./ganetimgr/manage.py migrate -RUN ./ganetimgr/manage.py collectstatic --noinput +COPY ganetimgr.nginx.conf /etc/nginx/nginx.conf +# nginx run inside the container EXPOSE 80 -EXPOSE 8000 -#ENTRYPOINT bash /run.sh -ENTRYPOINT nginx && ./ganetimgr/manage.py runserver 0.0.0.0:8080 +COPY entrypoint.sh / +# Set this as a CMD instead of ENTRYPOINT in order to be able to override it +CMD ["/entrypoint.sh"] diff --git a/README.md b/README.md index 43cfa5d..1540999 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,22 @@ ganetimgr-docker ================ -docker build -t grnet/ganetimgr . +# This creates a container running ganeti. +# KVM (the kernel module) must be present on the host +# so /dev/kvm will be passed to the container. +docker build -t grnet/ganeti ganeti/Dockerfile +docker run --privileged --name ganeti grnet/ganeti -docker run -p 80:80 grnet/ganetimgr:latest +# Runs ganetimgr in a container. +# Uses sqlite, no redis and no beanstalkd/watcher by default. +docker build -t grnet/ganetimgr Dockerfile +docker run --link ganeti:ganeti -e GANETIMGR_ADMIN_PASS= -p 80:80 --name ganetimgr grnet/ganetimgr -docker exec -ti ganetimgr /bin/bash +# watcher container - runs a beanstalk tube +# needs python deps and the ganetimgr project +docker build -t grnet/ganetimgr-watcher Dockerfile +docker run --link ganetimgr:ganetimgr --name ganetimgr-watcher grnet/ganetimgr-watcher -from django.contrib.auth.models import User; User.objects.create_superuser('admin', 'admin@example.com','12345') - +# To get shell access to any of the containers: +docker exec -ti /bin/bash diff --git a/beanstalkd.conf b/beanstalkd.conf deleted file mode 100644 index 4e2bd84..0000000 --- a/beanstalkd.conf +++ /dev/null @@ -1,9 +0,0 @@ -## Defaults for the beanstalkd init script, /etc/init.d/beanstalkd on -## Debian systems. Append ``-b /var/lib/beanstalkd'' for persistent -## storage. -BEANSTALKD_LISTEN_ADDR=0.0.0.0 -BEANSTALKD_LISTEN_PORT=11300 -DAEMON_OPTS="-l $BEANSTALKD_LISTEN_ADDR -p $BEANSTALKD_LISTEN_PORT" - -## Uncomment to enable startup during boot. -START=yes diff --git a/docker-compose.yml b/docker-compose.yml index 2ebb006..1fbcf1f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,24 +1,33 @@ version: "2" services: beanstalkd: image: agaveapi/beanstalkd database: image: mysql environment: MYSQL_ROOT_PASSWORD= MYSQL_USER=ganetimgr MYSQL_PASSWORD= MYSQL_DATABASE=ganetimgr worker: build: context: . dockerfile: Dockerfile links: - beanstalkd + - ganeti depends_on: - database - beanstalkd + extra_hosts: + - "ganeti:172.17.0.230" + + ganeti: + build: + context: ganeti/ + dockerfile: Dockerfile + privileged: true diff --git a/dumpdata.json b/dumpdata.json new file mode 100644 index 0000000..3dd989e --- /dev/null +++ b/dumpdata.json @@ -0,0 +1,18 @@ +[ + { + "fields": { + "description": "", + "disable_instance_creation": false, + "disabled": false, + "fast_create": false, + "hostname": "172.17.0.230", + "password": "test", + "port": 5080, + "slug": "172170230", + "use_gnt_network": true, + "username": "ganeti" + }, + "model": "ganeti.cluster", + "pk": 1 + } +] diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..722e876 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Create a dummy file, until this is no longer needed +touch templates/includes/analytics.html + + +/srv/ganetimgr/manage.py loaddata /dumpdata.json + + + + +# Django init commands +python manage.py syncdb --noinput -v0 --migrate +python manage.py collectstatic --noinput -v0 -l + + +if [ -n "$GANETIMGR_ADMIN_PASS" ]; then + echo "from django.contrib.auth.models import User; User.objects.create_superuser('admin', 'admin@example.com', '$GANETIMGR_ADMIN_PASS')" | python manage.py shell + echo "from django.contrib.auth.models import User; User.objects.create_user('user', 'user@example.com', '$GANETIMGR_ADMIN_PASS')" | python manage.py shell +fi + +# Start background services +sed -i 's/#START=yes/START=yes/' /etc/default/beanstalkd +/etc/init.d/beanstalkd start +/etc/init.d/redis-server start +/etc/init.d/nginx start + +mkdir /srv/logs +touch /srv/logs/gunicorn.log /srv/logs/access.log +tail -f /srv/logs/*.log & + +echo "Starting Gunicorn." +gunicorn ganetimgr.wsgi:application --name ganetimgr --bind 0.0.0.0:8000 --workers=1 --log-level=info --log-file=/srv/logs/gunicorn.log --access-logfile=/srv/logs/access.log "$@" diff --git a/ganeti/Dockerfile b/ganeti/Dockerfile new file mode 100644 index 0000000..213d75b --- /dev/null +++ b/ganeti/Dockerfile @@ -0,0 +1,14 @@ +FROM debian:jessie + +RUN apt-get update -q2 && DEBIAN_FRONTEND=noninteractive apt-get install --quiet --no-install-recommends --yes ganeti ganeti-instance-debootstrap patch qemu-kvm + +# SNF-IMAGE install +#echo "deb http://apt.dev.grnet.gr jessie/" >> /etc/apt/sources.list.d/snf-image.list +#curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add - + +COPY gnt_deboo_losetup.patch / + +EXPOSE 5080 + +COPY entrypoint.sh / +ENTRYPOINT ["/entrypoint.sh"] diff --git a/ganeti/entrypoint.sh b/ganeti/entrypoint.sh new file mode 100755 index 0000000..dcbaf33 --- /dev/null +++ b/ganeti/entrypoint.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +echo "172.17.0.230 ganeti.example.com" >> /etc/hosts + +cd /usr/share/ganeti/os/debootstrap/ && patch < /gnt_deboo_losetup.patch > /dev/null + +mkdir /srv/ganeti && echo "/srv/ganeti" > /etc/ganeti/file-storage-paths + +sed -i "s/RAPI_ARGS=\"-b 127.0.0.1 --require-authentication\"/RAPI_ARGs=\"--require-authentication\"/" /etc/default/ganeti +#sed -i "s/WCONFD_ARGS=\"\"/WCONFD_ARGS=\"-f\"/" /etc/default/ganeti + +mkdir /var/lib/ganeti/rapi/ +chown gnt-rapi:gnt-masterd /var/lib/ganeti/rapi/ +echo "ganeti test write" >> /var/lib/ganeti/rapi/users + +/usr/sbin/gnt-cluster init --enabled-hypervisors=kvm --enabled-disk-templates=file --file-storage-dir=/srv/ganeti --master-netdev=eth0 --no-ssh-init --no-etc-hosts -H kvm:kernel_path='' ganeti.example.com + +mkdir /var/run/sshd +/usr/sbin/sshd -D diff --git a/ganeti/gnt_deboo_losetup.patch b/ganeti/gnt_deboo_losetup.patch new file mode 100644 index 0000000..9d9fa10 --- /dev/null +++ b/ganeti/gnt_deboo_losetup.patch @@ -0,0 +1,65 @@ +From: Jose A. Lopes +Date: Fri, 24 Jan 2014 09:23:01 +0000 (+0100) +Subject: Replace 'losetup' flag '-s' with '--show' +X-Git-Tag: v0.15~4 +X-Git-Url: http://git.ganeti.org/?p=instance-debootstrap.git;a=commitdiff_plain;h=913c6e4222969470796729cf188bb79a78635d8a + +Replace 'losetup' flag '-s' with '--show' + +This fixes issue 690. + +Signed-off-by: Jose A. Lopes +Reviewed-by: Klaus Aehlig +--- + +diff --git a/create b/create +index c276b04..6565176 100755 +--- a/create ++++ b/create +@@ -36,7 +36,7 @@ CACHE_FILE="$CACHE_DIR/cache-${SUITE}-${DPKG_ARCH}.tar" + # This is needed for file disks. + if [ ! -b $blockdev ]; then + ORIGINAL_BLOCKDEV=$blockdev +- blockdev=$(losetup -sf $blockdev) ++ blockdev=$(losetup --show -f $blockdev) + CLEANUP+=("losetup -d $blockdev") + fi + +diff --git a/export b/export +index 46aa74c..8941621 100755 +--- a/export ++++ b/export +@@ -25,7 +25,7 @@ set -e + # This is needed for file disks. + if [ ! -b $blockdev ]; then + ORIGINAL_BLOCKDEV=$blockdev +- blockdev=$(losetup -sf $blockdev) ++ blockdev=$(losetup --show -f $blockdev) + CLEANUP+=("losetup -d $blockdev") + fi + +diff --git a/import b/import +index 2d9b58e..a69759d 100755 +--- a/import ++++ b/import +@@ -25,7 +25,7 @@ set -e + # This is needed for file disks. + if [ ! -b $blockdev ]; then + ORIGINAL_BLOCKDEV=$blockdev +- blockdev=$(losetup -sf $blockdev) ++ blockdev=$(losetup --show -f $blockdev) + CLEANUP+=("losetup -d $blockdev") + fi + +diff --git a/rename b/rename +index 652d6b7..81bf8dd 100755 +--- a/rename ++++ b/rename +@@ -28,7 +28,7 @@ CLEANUP+=("rmdir $TMPDIR") + # This is needed for file disks. + if [ ! -b $blockdev ]; then + ORIGINAL_BLOCKDEV=$blockdev +- blockdev=$(losetup -sf $blockdev) ++ blockdev=$(losetup --show -f $blockdev) + CLEANUP+=("losetup -d $blockdev") + fi diff --git a/ganetimgr.nginx.conf b/ganetimgr.nginx.conf index 87b57e8..5552b58 100644 --- a/ganetimgr.nginx.conf +++ b/ganetimgr.nginx.conf @@ -1,26 +1,26 @@ worker_processes 1; events { worker_connections 1024; } http { include mime.types; server { listen 80; server_name _; access_log /dev/stdout; error_log /dev/stdout info; location /static { alias /srv/ganetimgr/static; } location / { - proxy_pass http://localhost:8080; + proxy_pass http://localhost:8000; include proxy_params; } } } diff --git a/run.sh b/run.sh deleted file mode 100644 index 6caae93..0000000 --- a/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -/etc/init.d/beanstalkd start -/etc/init.d/redis-server start -/etc/init.d/gunicorn start diff --git a/settings.py b/settings.py index 073af51..c067e8e 100644 --- a/settings.py +++ b/settings.py @@ -1,336 +1,308 @@ # -*- coding: utf-8 -*- vim:fileencoding=utf-8: + +# shortcuts to create relative paths import os -import dj_database_url BASE_DIR = os.path.dirname(os.path.dirname(__file__)) PROJECT_DIR = os.path.join(BASE_DIR, 'ganetimgr') +# helper function to get database connection from env.var +import dj_database_url DATABASES= {} - DATABASES['default'] = dj_database_url.config(default='sqlite:////%s' % os.path.join(BASE_DIR, 'db.sqlite3')) -DEBUG = True +# +DEBUG = os.environ.get('DJANGO_DEBUG', True) +TIME_ZONE = os.environ.get('DJANGO_TIMEZONE','Europe/Athens') ALLOWED_HOSTS=["*"] TEMPLATE_DEBUG = DEBUG - SITE_ID = 1 +ADMINS = ( ('John Doe', 'john@example.com'),) +MANAGERS = ADMINS +SECRET_KEY = '' -# Time zone & localization -TIME_ZONE = 'Europe/Athens' +# Locale settings +# We provide English and Greek translations _ = lambda s: s - +#LANGUAGE_CODE = 'en-us' LANGUAGES = ( # ('el', u'Ελληνικά'), ('en', _('English')), ) - -#LANGUAGE_CODE = 'en-us' - LOCALE_PATHS = ( os.path.join(BASE_DIR, 'locale'), ) + DATE_FORMAT = "d/m/Y H:i" DATETIME_FORMAT = "d/m/Y H:i" # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. USE_I18N = True USE_L10N = True MEDIA_ROOT = '' MEDIA_URL = '' # List of callables that know how to import templates from various sources. TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ) MIDDLEWARE_CLASSES = ( 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'middleware.ForceLogout.ForceLogoutMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware', 'middleware.UserMessages.UserMessageMiddleware', ) STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'static') - -STATICFILES_DIRS = ( -) - +STATICFILES_DIRS = () STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ) ROOT_URLCONF = 'ganetimgr.urls' TEMPLATE_DIRS = ( os.path.join(BASE_DIR, 'templates'), ) INSTALLED_APPS = ( 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.flatpages', 'django.contrib.messages', 'django.contrib.admin', 'django.contrib.staticfiles', 'registration', 'django_markdown', 'accounts', 'south', 'ganeti', 'apply', 'notifications', 'stats', 'auditlog', -# 'oauth2_provider', -# 'corsheaders', ) # Caching is a vital part of ganetimgr. # If you deploy more than one ganetimgr instances on the same server, # and want to use Redis for both, make sure you select a different db for each instance # Warning!!! Redis db should ALWAYS be an integer, denoting db index. # If memcache is your preferred cache, then select: #CACHE_BACKEND="redis_cache.cache://127.0.0.1:6379?timeout=1500" #CACHES = { # 'default': { # 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', # } # 'default': { # 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', # 'LOCATION': '127.0.0.1:11211', # 'TIMEOUT': 1, # } #} LOGIN_URL = '/user/login' LOGIN_REDIRECT_URL = '/' TEMPLATE_CONTEXT_PROCESSORS = ( "django.contrib.auth.context_processors.auth", "django.core.context_processors.debug", "django.core.context_processors.i18n", "django.core.context_processors.media", "context.pending_notifications.notify", "context.session_remaining.seconds", "context.global_vars.settings_vars", "django.core.context_processors.request", "django.contrib.messages.context_processors.messages" ) -EMAIL_HOST = "127.0.0.1" -EMAIL_PORT = 25 +#EMAIL_HOST = "127.0.0.1" +#EMAIL_PORT = 25 USE_X_FORWARDED_HOST = True # Auth stuff # If you plan to deploy LDAP modify according to your needs AUTHENTICATION_BACKENDS = ( #'django_auth_ldap.backend.LDAPBackend', 'django.contrib.auth.backends.ModelBackend', ) #import ldap #from django_auth_ldap.config import LDAPSearch #AUTH_LDAP_BIND_DN = "" #AUTH_LDAP_BIND_PASSWORD = "" #AUTH_LDAP_SERVER_URI = "ldap://ldap.example.com" #AUTH_LDAP_START_TLS = True #AUTH_LDAP_USER_SEARCH = LDAPSearch("ou=People,dc=dept,dc=example,dc=com", # ldap.SCOPE_SUBTREE, "(uid=%(user)s)") #AUTH_LDAP_USER_ATTR_MAP = { # "first_name": "givenName", # "last_name": "sn", # "email": "mail" # } ACCOUNT_ACTIVATION_DAYS = 10 AUTH_PROFILE_MODULE = 'accounts.UserProfile' SESSION_EXPIRE_AT_BROWSER_CLOSE = True #SESSION_ENGINE = "django.contrib.sessions.backends.cache" SESSION_COOKIE_AGE = 10800 IDLE_ACCOUNT_NOTIFICATION_DAYS = '180' # Number of days that hash verification is active INSTANCE_ACTION_ACTIVE_DAYS = 7 # This works for our GRNET NOC Jira installation. Default is False HELPDESK_INTEGRATION_JAVASCRIPT_URL = "" HELPDESK_INTEGRATION_JAVASCRIPT_PARAMS = { 'customfield_11551': 'tier-1' } COLLECTD_URL = "http://stats.example.com" # Graphs nodata image NODATA_IMAGE = 'static/nodata.gif' SERVER_MONITORING_URL = 'https://monitoring.example.com' import _version SW_VERSION = _version.VERSION NODATA_IMAGE = 'static/nodata.gif' WHITELIST_IP_MAX_SUBNET_V4 = 26 WHITELIST_IP_MAX_SUBNET_V6 = 64 # RSS Feed for the login page FEED_URL = "" # Choose whether to support websockets console or not. WEBSOCK_VNC_ENABLED = True # This is meant to be used with twistednovncauthproxy # twistd --pidfile=/tmp/proxy.pid -n vncap -c tcp:8888:interface=0.0.0.0 NOVNC_PROXY = "vnc.proxy.com:8888" NOVNC_USE_TLS = True BRANDING = { "SERVICE_PROVIDED_BY": { "NAME": "EXAMPLE", "URL": "//example.dot.com", "SOCIAL_NETWORKS": [ { "URL": "https://facebook.com/", "FONT_AWESOME_NAME": "fa-facebook", "FONT_COLOR": "#3b5998" }, { "URL": "https://twitter.com/", "FONT_AWESOME_NAME": "fa-twitter", "FONT_COLOR": "#00acee" } ] }, "VIDEO": "", # iframe url "LOGO": "/static/ganetimgr/img/logo.png", "FAVICON": "/static/ganetimgr/img/favicon.ico", "MOTTO": "virtual private servers", "FOOTER_ICONS_IFRAME": False, # show the administrative contact # option when creating a new vm "SHOW_ADMINISTRATIVE_FORM": True, "SHOW_ORGANIZATION_FORM": True, "TITLE": "GanetiMGR", } # Set the email subject prefix: EMAIL_SUBJECT_PREFIX = "[GANETIMGR SERVICE] " SERVER_EMAIL = "no-reply@example.com" DEFAULT_FROM_EMAIL = "no-reply@example.com" # Flatpages manipulation. Show or hide flatpages links in page. FLATPAGES = { "INFO": True, "TOS": True, "FAQ": True, } # Get a recaptcha key RECAPTCHA_PUBLIC_KEY = '' RECAPTCHA_PRIVATE_KEY = '' RECAPTCHA_USE_SSL = True MARKDOWN_EDITOR_SKIN = 'simple' ######################### # # # Ganeti # # # ######################### # Select your ganetimgr prefix. This is applied in the tags # of the instances. You could leave it as it is or set your own, # eg. GANETI_TAG_PREFIX = "vmservice" GANETI_TAG_PREFIX = "ganetimgr" RAPI_CONNECT_TIMEOUT = 8 RAPI_RESPONSE_TIMEOUT = 15 # List of operating system images you provide... OPERATING_SYSTEMS = { "none": { "description": "No operating system", "provider": "noop", "osparams": {}, "ssh_key_param": "", }, + "debootstrap": { + "description": "Debootstrap", + "provider": "debootstrap+default", + "osparams": {}, + "ssh_key_param": "", + }, + } # the urls of the available os images OPERATING_SYSTEMS_URLS = ['http://example.com/images'] SNF_OPERATING_SYSTEMS_URLS = ['http://example.com/snf-images/'] # the provider and ssh key param # We assume that they have the same configuration OPERATING_SYSTEMS_PROVIDER = 'image+default' OPERATING_SYSTEMS_SSH_KEY_PARAM = 'img_ssh_key_url' SNF_IMG_PROPERTIES = { "SWAP": "2:512" } SNF_IMG_PASSWD = "example-passphrase" ######################### # # # Auditlog # # # ######################### # this option sets the amount of days old an audit entry has to be # in order to be shown in audit log page # '0' means show all entries AUDIT_ENTRIES_LAST_X_DAYS = 10 -# Instance specific django config. -ADMINS = ( - ('John Doe', 'john@example.com'), -) -MANAGERS = ADMINS - -#DATABASES = { -# 'default': { -# 'ENGINE': 'django.db.backends.', # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. -# 'NAME': '', # Or path to database file if using sqlite3. -# 'USER': '', # Not used with sqlite3. -# 'PASSWORD': '', # Not used with sqlite3. -# 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. -# 'PORT': '', # Set to empty string for default. Not used with sqlite3. -# 'OPTIONS': {'init_command': 'SET storage_engine=MYISAM;'} -# } -#} - -# Make this unique, and don't share it with anybody. -SECRET_KEY = '' - - -# OAUTH2 -if 'oauth2_provider' in INSTALLED_APPS and 'corsheaders' in INSTALLED_APPS: - OAUTH2_PROVIDER = { - 'ACCESS_TOKEN_EXPIRE_SECONDS': 60 * 60 * 24 * 7 * 10, - 'SCOPES': { - 'read': 'Read scope', - }, - 'CLIENT_ID_GENERATOR_CLASS': 'oauth2_provider.generators.ClientIdGenerator', - } - CORS_ORIGIN_ALLOW_ALL = True - MIDDLEWARE_CLASSES += ('corsheaders.middleware.CorsMiddleware',) - diff --git a/watcher/Dockerfile b/watcher/Dockerfile new file mode 100644 index 0000000..ff6c19a --- /dev/null +++ b/watcher/Dockerfile @@ -0,0 +1,39 @@ +####################################### +#Dockerfile to build a ganetimgr image# +#Uses Deban packages instead of pip # +####################################### + +# We use wheezy as a base (for now) +FROM debian:wheezy +MAINTAINER GRNET_NOC +ENV GANETIMGR_UPSTREAM_URL https://github.com/grnet/ganetimgr.git + +# First layer - only system packages, nothing from the stack +RUN apt-get update -q2 && DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -q2 git procps apt-utils + +# Django and rest of python dependencies for the project +RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -q2 python-django python-redis python-mysqldb python-django-south python-django-registration python-paramiko python-simplejson python-daemon python-setproctitle python-pycurl python-recaptcha python-ipaddr python-bs4 python-requests python-markdown python-gevent + +# Daemon dependencies +RUN DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -q2 beanstalkd + +# This is a workaound for a project dependency that has no Debian package +ADD python-django-markdown_0.6.1-1_all.deb / +RUN dpkg -i /python-django-markdown_0.6.1-1_all.deb +# Can be removed when the commit that removes the dep is on master + +# Get the repository and switch context inside it +ENV GANETIMGR_INSTALLDIR=/srv/ganetimgr +RUN git clone --quiet $GANETIMGR_UPSTREAM_URL $GANETIMGR_INSTALLDIR +WORKDIR $GANETIMGR_INSTALLDIR + +# Predifined Settings for use inside the container +COPY settings.py $GANETIMGR_INSTALLDIR/ganetimgr/settings.py +# Helper function to get the db connection info from envvars +COPY dj_database_url.py ganetimgr/dj_database_url.py + +# nginx run inside the container +EXPOSE 11300 +COPY entrypoint.sh / +# Set this as a CMD instead of ENTRYPOINT in order to be able to override it +CMD ["/entrypoint.sh"] diff --git a/watcher/dj_database_url.py b/watcher/dj_database_url.py new file mode 100644 index 0000000..e269e9e --- /dev/null +++ b/watcher/dj_database_url.py @@ -0,0 +1,129 @@ +# -*- coding: utf-8 -*- + +import os + +try: + import urlparse +except ImportError: + import urllib.parse as urlparse + + +# Register database schemes in URLs. +urlparse.uses_netloc.append('postgres') +urlparse.uses_netloc.append('postgresql') +urlparse.uses_netloc.append('pgsql') +urlparse.uses_netloc.append('postgis') +urlparse.uses_netloc.append('mysql') +urlparse.uses_netloc.append('mysql2') +urlparse.uses_netloc.append('mysqlgis') +urlparse.uses_netloc.append('mysql-connector') +urlparse.uses_netloc.append('spatialite') +urlparse.uses_netloc.append('sqlite') +urlparse.uses_netloc.append('oracle') +urlparse.uses_netloc.append('oraclegis') +urlparse.uses_netloc.append('redshift') + +DEFAULT_ENV = 'DATABASE_URL' + +SCHEMES = { + 'postgres': 'django.db.backends.postgresql_psycopg2', + 'postgresql': 'django.db.backends.postgresql_psycopg2', + 'pgsql': 'django.db.backends.postgresql_psycopg2', + 'postgis': 'django.contrib.gis.db.backends.postgis', + 'mysql': 'django.db.backends.mysql', + 'mysql2': 'django.db.backends.mysql', + 'mysqlgis': 'django.contrib.gis.db.backends.mysql', + 'mysql-connector': 'mysql.connector.django', + 'spatialite': 'django.contrib.gis.db.backends.spatialite', + 'sqlite': 'django.db.backends.sqlite3', + 'oracle': 'django.db.backends.oracle', + 'oraclegis': 'django.contrib.gis.db.backends.oracle', + 'redshift': 'django_redshift_backend', +} + + +def config(env=DEFAULT_ENV, default=None, engine=None, conn_max_age=0): + """Returns configured DATABASE dictionary from DATABASE_URL.""" + + config = {} + + s = os.environ.get(env, default) + + if s: + config = parse(s, engine, conn_max_age) + + return config + + +def parse(url, engine=None, conn_max_age=0): + """Parses a database URL.""" + + if url == 'sqlite://:memory:': + # this is a special case, because if we pass this URL into + # urlparse, urlparse will choke trying to interpret "memory" + # as a port number + return { + 'ENGINE': SCHEMES['sqlite'], + 'NAME': ':memory:' + } + # note: no other settings are required for sqlite + + # otherwise parse the url as normal + config = {} + + url = urlparse.urlparse(url) + + # Split query strings from path. + path = url.path[1:] + if '?' in path and not url.query: + path, query = path.split('?', 2) + else: + path, query = path, url.query + query = urlparse.parse_qs(query) + + # If we are using sqlite and we have no path, then assume we + # want an in-memory database (this is the behaviour of sqlalchemy) + if url.scheme == 'sqlite' and path == '': + path = ':memory:' + + # Handle postgres percent-encoded paths. + hostname = url.hostname or '' + if '%2f' in hostname.lower(): + hostname = hostname.replace('%2f', '/').replace('%2F', '/') + + # Update with environment configuration. + config.update({ + 'NAME': urlparse.unquote(path or ''), + 'USER': urlparse.unquote(url.username or ''), + 'PASSWORD': urlparse.unquote(url.password or ''), + 'HOST': hostname, + 'PORT': url.port or '', + 'CONN_MAX_AGE': conn_max_age, + }) + + # Lookup specified engine. + engine = SCHEMES[url.scheme] if engine is None else engine + + # Pass the query string into OPTIONS. + options = {} + for key, values in query.items(): + if url.scheme == 'mysql' and key == 'ssl-ca': + options['ssl'] = {'ca': values[-1]} + continue + + options[key] = values[-1] + + # Support for Postgres Schema URLs + if 'currentSchema' in options and engine in ( + 'django.db.backends.postgresql_psycopg2', + 'django_redshift_backend', + ): + options['options'] = '-c search_path={0}'.format(options.pop('currentSchema')) + + if options: + config['OPTIONS'] = options + + if engine: + config['ENGINE'] = engine + + return config diff --git a/watcher/entrypoint.sh b/watcher/entrypoint.sh new file mode 100755 index 0000000..796ce7a --- /dev/null +++ b/watcher/entrypoint.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +# Create a dummy file, until this is no longer needed +touch templates/includes/analytics.html +mkdir /var/log/ganetimgr/ + +# Django init commands +python manage.py syncdb --noinput -v0 --migrate +python manage.py collectstatic --noinput -v0 -l + +# Start background services +sed -i 's/#START=yes/START=yes/' /etc/default/beanstalkd +/etc/init.d/beanstalkd start > /dev/null + +/srv/ganetimgr/watcher.py --foreground --log-file - diff --git a/watcher/python-django-markdown_0.6.1-1_all.deb b/watcher/python-django-markdown_0.6.1-1_all.deb new file mode 100644 index 0000000..1ad0720 Binary files /dev/null and b/watcher/python-django-markdown_0.6.1-1_all.deb differ diff --git a/settings.py b/watcher/settings.py similarity index 85% copy from settings.py copy to watcher/settings.py index 073af51..a759146 100644 --- a/settings.py +++ b/watcher/settings.py @@ -1,336 +1,308 @@ # -*- coding: utf-8 -*- vim:fileencoding=utf-8: + +# shortcuts to create relative paths import os -import dj_database_url BASE_DIR = os.path.dirname(os.path.dirname(__file__)) PROJECT_DIR = os.path.join(BASE_DIR, 'ganetimgr') +# helper function to get database connection from env.var +import dj_database_url DATABASES= {} - DATABASES['default'] = dj_database_url.config(default='sqlite:////%s' % os.path.join(BASE_DIR, 'db.sqlite3')) -DEBUG = True +# +DEBUG = os.environ.get('DJANGO_DEBUG', True) +TIME_ZONE = os.environ.get('DJANGO_TIMEZONE','Europe/Athens') ALLOWED_HOSTS=["*"] TEMPLATE_DEBUG = DEBUG - SITE_ID = 1 +ADMINS = ( ('John Doe', 'john@example.com'),) +MANAGERS = ADMINS +SECRET_KEY = '' -# Time zone & localization -TIME_ZONE = 'Europe/Athens' +# Locale settings +# We provide English and Greek translations _ = lambda s: s - +#LANGUAGE_CODE = 'en-us' LANGUAGES = ( # ('el', u'Ελληνικά'), ('en', _('English')), ) - -#LANGUAGE_CODE = 'en-us' - LOCALE_PATHS = ( os.path.join(BASE_DIR, 'locale'), ) + DATE_FORMAT = "d/m/Y H:i" DATETIME_FORMAT = "d/m/Y H:i" # If you set this to False, Django will make some optimizations so as not # to load the internationalization machinery. USE_I18N = True USE_L10N = True MEDIA_ROOT = '' MEDIA_URL = '' # List of callables that know how to import templates from various sources. TEMPLATE_LOADERS = ( 'django.template.loaders.filesystem.Loader', 'django.template.loaders.app_directories.Loader', ) MIDDLEWARE_CLASSES = ( 'django.middleware.common.CommonMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'middleware.ForceLogout.ForceLogoutMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.middleware.locale.LocaleMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.contrib.flatpages.middleware.FlatpageFallbackMiddleware', 'middleware.UserMessages.UserMessageMiddleware', ) STATIC_URL = '/static/' STATIC_ROOT = os.path.join(BASE_DIR, 'static') - -STATICFILES_DIRS = ( -) - +STATICFILES_DIRS = () STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', ) ROOT_URLCONF = 'ganetimgr.urls' TEMPLATE_DIRS = ( os.path.join(BASE_DIR, 'templates'), ) INSTALLED_APPS = ( 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', 'django.contrib.flatpages', 'django.contrib.messages', 'django.contrib.admin', 'django.contrib.staticfiles', 'registration', 'django_markdown', 'accounts', 'south', 'ganeti', 'apply', 'notifications', 'stats', 'auditlog', -# 'oauth2_provider', -# 'corsheaders', ) # Caching is a vital part of ganetimgr. # If you deploy more than one ganetimgr instances on the same server, # and want to use Redis for both, make sure you select a different db for each instance # Warning!!! Redis db should ALWAYS be an integer, denoting db index. # If memcache is your preferred cache, then select: #CACHE_BACKEND="redis_cache.cache://127.0.0.1:6379?timeout=1500" #CACHES = { # 'default': { # 'BACKEND': 'django.core.cache.backends.dummy.DummyCache', # } # 'default': { # 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', # 'LOCATION': '127.0.0.1:11211', # 'TIMEOUT': 1, # } #} LOGIN_URL = '/user/login' LOGIN_REDIRECT_URL = '/' TEMPLATE_CONTEXT_PROCESSORS = ( "django.contrib.auth.context_processors.auth", "django.core.context_processors.debug", "django.core.context_processors.i18n", "django.core.context_processors.media", "context.pending_notifications.notify", "context.session_remaining.seconds", "context.global_vars.settings_vars", "django.core.context_processors.request", "django.contrib.messages.context_processors.messages" ) -EMAIL_HOST = "127.0.0.1" -EMAIL_PORT = 25 +#EMAIL_HOST = "127.0.0.1" +#EMAIL_PORT = 25 USE_X_FORWARDED_HOST = True # Auth stuff # If you plan to deploy LDAP modify according to your needs AUTHENTICATION_BACKENDS = ( #'django_auth_ldap.backend.LDAPBackend', 'django.contrib.auth.backends.ModelBackend', ) #import ldap #from django_auth_ldap.config import LDAPSearch #AUTH_LDAP_BIND_DN = "" #AUTH_LDAP_BIND_PASSWORD = "" #AUTH_LDAP_SERVER_URI = "ldap://ldap.example.com" #AUTH_LDAP_START_TLS = True #AUTH_LDAP_USER_SEARCH = LDAPSearch("ou=People,dc=dept,dc=example,dc=com", # ldap.SCOPE_SUBTREE, "(uid=%(user)s)") #AUTH_LDAP_USER_ATTR_MAP = { # "first_name": "givenName", # "last_name": "sn", # "email": "mail" # } ACCOUNT_ACTIVATION_DAYS = 10 AUTH_PROFILE_MODULE = 'accounts.UserProfile' SESSION_EXPIRE_AT_BROWSER_CLOSE = True #SESSION_ENGINE = "django.contrib.sessions.backends.cache" SESSION_COOKIE_AGE = 10800 IDLE_ACCOUNT_NOTIFICATION_DAYS = '180' # Number of days that hash verification is active INSTANCE_ACTION_ACTIVE_DAYS = 7 # This works for our GRNET NOC Jira installation. Default is False HELPDESK_INTEGRATION_JAVASCRIPT_URL = "" HELPDESK_INTEGRATION_JAVASCRIPT_PARAMS = { 'customfield_11551': 'tier-1' } COLLECTD_URL = "http://stats.example.com" # Graphs nodata image NODATA_IMAGE = 'static/nodata.gif' SERVER_MONITORING_URL = 'https://monitoring.example.com' import _version SW_VERSION = _version.VERSION NODATA_IMAGE = 'static/nodata.gif' WHITELIST_IP_MAX_SUBNET_V4 = 26 WHITELIST_IP_MAX_SUBNET_V6 = 64 # RSS Feed for the login page FEED_URL = "" # Choose whether to support websockets console or not. WEBSOCK_VNC_ENABLED = True # This is meant to be used with twistednovncauthproxy # twistd --pidfile=/tmp/proxy.pid -n vncap -c tcp:8888:interface=0.0.0.0 NOVNC_PROXY = "vnc.proxy.com:8888" NOVNC_USE_TLS = True BRANDING = { "SERVICE_PROVIDED_BY": { "NAME": "EXAMPLE", "URL": "//example.dot.com", "SOCIAL_NETWORKS": [ { "URL": "https://facebook.com/", "FONT_AWESOME_NAME": "fa-facebook", "FONT_COLOR": "#3b5998" }, { "URL": "https://twitter.com/", "FONT_AWESOME_NAME": "fa-twitter", "FONT_COLOR": "#00acee" } ] }, "VIDEO": "", # iframe url "LOGO": "/static/ganetimgr/img/logo.png", "FAVICON": "/static/ganetimgr/img/favicon.ico", "MOTTO": "virtual private servers", "FOOTER_ICONS_IFRAME": False, # show the administrative contact # option when creating a new vm "SHOW_ADMINISTRATIVE_FORM": True, "SHOW_ORGANIZATION_FORM": True, "TITLE": "GanetiMGR", } # Set the email subject prefix: EMAIL_SUBJECT_PREFIX = "[GANETIMGR SERVICE] " SERVER_EMAIL = "no-reply@example.com" DEFAULT_FROM_EMAIL = "no-reply@example.com" # Flatpages manipulation. Show or hide flatpages links in page. FLATPAGES = { "INFO": True, "TOS": True, "FAQ": True, } # Get a recaptcha key RECAPTCHA_PUBLIC_KEY = '' RECAPTCHA_PRIVATE_KEY = '' RECAPTCHA_USE_SSL = True MARKDOWN_EDITOR_SKIN = 'simple' ######################### # # # Ganeti # # # ######################### # Select your ganetimgr prefix. This is applied in the tags # of the instances. You could leave it as it is or set your own, # eg. GANETI_TAG_PREFIX = "vmservice" GANETI_TAG_PREFIX = "ganetimgr" RAPI_CONNECT_TIMEOUT = 8 RAPI_RESPONSE_TIMEOUT = 15 # List of operating system images you provide... OPERATING_SYSTEMS = { "none": { "description": "No operating system", "provider": "noop", "osparams": {}, "ssh_key_param": "", }, + "debootstrap": { + "description": "Debootstrap", + "provider": "debootstrap+default", + "osparams": { }, + "ssh_key_param": "", + }, + } # the urls of the available os images OPERATING_SYSTEMS_URLS = ['http://example.com/images'] SNF_OPERATING_SYSTEMS_URLS = ['http://example.com/snf-images/'] # the provider and ssh key param # We assume that they have the same configuration OPERATING_SYSTEMS_PROVIDER = 'image+default' OPERATING_SYSTEMS_SSH_KEY_PARAM = 'img_ssh_key_url' SNF_IMG_PROPERTIES = { "SWAP": "2:512" } SNF_IMG_PASSWD = "example-passphrase" ######################### # # # Auditlog # # # ######################### # this option sets the amount of days old an audit entry has to be # in order to be shown in audit log page # '0' means show all entries AUDIT_ENTRIES_LAST_X_DAYS = 10 -# Instance specific django config. -ADMINS = ( - ('John Doe', 'john@example.com'), -) -MANAGERS = ADMINS - -#DATABASES = { -# 'default': { -# 'ENGINE': 'django.db.backends.', # 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'. -# 'NAME': '', # Or path to database file if using sqlite3. -# 'USER': '', # Not used with sqlite3. -# 'PASSWORD': '', # Not used with sqlite3. -# 'HOST': '', # Set to empty string for localhost. Not used with sqlite3. -# 'PORT': '', # Set to empty string for default. Not used with sqlite3. -# 'OPTIONS': {'init_command': 'SET storage_engine=MYISAM;'} -# } -#} - -# Make this unique, and don't share it with anybody. -SECRET_KEY = '' - - -# OAUTH2 -if 'oauth2_provider' in INSTALLED_APPS and 'corsheaders' in INSTALLED_APPS: - OAUTH2_PROVIDER = { - 'ACCESS_TOKEN_EXPIRE_SECONDS': 60 * 60 * 24 * 7 * 10, - 'SCOPES': { - 'read': 'Read scope', - }, - 'CLIENT_ID_GENERATOR_CLASS': 'oauth2_provider.generators.ClientIdGenerator', - } - CORS_ORIGIN_ALLOW_ALL = True - MIDDLEWARE_CLASSES += ('corsheaders.middleware.CorsMiddleware',) -