Merge branch 'celery' into development

.gitlab-ci.yml

@@ -18,10 +18,10 @@ check:test:
     - python manage.py test
 
 deploy:dev:
-  image: registry.blitzhub.io/docker
-  stage: deploy
   only:
     - development
+  image: registry.blitzhub.io/docker
+  stage: deploy
   services:
     - name: registry.blitzhub.io/docker:dind
       alias: docker
@@ -37,11 +37,11 @@ deploy:dev:
     - check:lint
     - check:test
 
-deploy:prod:
-  image: registry.blitzhub.io/docker
-  stage: deploy
+deploy:prod:  
   only:
     - production
+  image: registry.blitzhub.io/docker
+  stage: deploy
   when: manual
   services:
     - name: registry.blitzhub.io/docker:dind

docker-compose.yml

@@ -47,6 +47,27 @@ services:
     hostname: redis
     networks:
       - redis_net
+  celery:
+    restart: always
+    build: .
+    command: celery -A core worker -l info
+    volumes:
+      - ./:/usr/src/app/
+    depends_on:
+      - redis
+      - db
+    networks:
+      - redis_net
+  celery-beat:
+    build: .
+    command: celery -A core beat -l info
+    volumes:
+      - ./:/usr/src/app/
+    depends_on:
+      - redis
+      - db
+    networks:
+      - redis_net
 
 networks:
   db_net:

docker-entrypoint.sh

@@ -3,6 +3,12 @@
 python3 manage.py migrate --run-syncdb
 python3 manage.py createfixtures
 python3 manage.py cleanuptasks
-# sh import-data.sh
+
+file="docker-entrypoint.lock"
+# exit if entrypoint.lock exists to prevent new import of data every time docker is restarted
+if ! test -f "$file"; then
+    sh import-data.sh
+    touch $file
+fi
 
 /usr/bin/supervisord -c "/etc/supervisor/conf.d/supervisord.conf"

netex/settings.py

@@ -12,6 +12,9 @@ https://docs.djangoproject.com/en/3.0/ref/settings/
 
 import os
 
+# celery beat
+from celery.schedules import crontab
+
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
@@ -169,3 +172,13 @@ CACHES = {
         }
     }
 }
+
+CELERY_BROKER_URL = "redis://redis:6379"
+CELERY_RESULT_BACKEND = "redis://redis:6379"
+
+CELERY_BEAT_SCHEDULE = {
+    "update_db": {
+        "task": "control.celery_tasks.task_update_db_from_nedrex",
+        "schedule": crontab(minute="*/3"),
+    }
+}

requirements.txt

@@ -28,4 +28,5 @@ rq==1.3.0
 six==1.15.0
 sqlparse==0.3.1
 urllib3==1.25.10
-sqlalchemy==1.3.23
\ No newline at end of file
+sqlalchemy==1.3.23
+celery==5.1.2
\ No newline at end of file