############################################################################## ### push to dockerhub export DOCKER_ID_USER="dblyon" docker login docker tag agotool dblyon/agotool docker push dblyon/agotool ############################################################################## ### installation on red hat virtual server # copy static data to server (can be deleted after copying it to named volume) rsync -avr /Users/dblyon/modules/cpr/agotool/data david@10.34.6.24:/data_from_ody/ # pull github repo git clone ... or cd /var/www/agotool git pull origin docker # change preload to False in /var/www/agotool/app/python/variables.py, PRELOAD = False vim /var/www/agotool/app/python/variables.py # build the images docker-compose build # copy data to named volume (spin up another container that deletes itself after it is done) docker run --rm -it --volume /data_from_ody:/mounted_data --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp rsync -avr /mounted_data/data /agotool_data/ # delete data if needed rm -rf /data_from_ody # download newest resources docker run --rm -it --name update --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp python ./app/python/update_manager.py # test DB docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /agotool_data/data/PostgreSQL/copy_from_file_and_index_DELETE.psql docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /agotool_data/data/PostgreSQL/drop_and_rename.psql # populate DB for real docker exec -it agotool_db_1 psql -U postgres -d agotool -f /agotool_data/data/PostgreSQL/copy_from_file_and_index.psql docker exec -it agotool_db_1 psql -U postgres -d agotool -f /agotool_data/data/PostgreSQL/drop_and_rename.psql # change "preload" to True, change "skip_slow_downloads" to False, change "debug" to False vim /var/www/agotool/app/python/variables.py --> tadaaa it should work now ### scale a service like the flask-app docker-compose up -d --scale flaskapp=2 ############################################################################## ##### Tree structure of local version of repo . ├── app │ ├── conf.d │ ├── python │ └── static │ ├── css │ ├── js │ └── templates └── data ├── Background_Reference_Proteomes ├── PostgreSQL │ ├── downloads │ ├── static │ └── tables │ └── test ├── exampledata ├── logs └── session ##### on virtual server # named volume /agotool_data/ └── data ├── Background_Reference_Proteomes ├── PostgreSQL │ ├── downloads │ ├── static │ └── tables │ └── test ├── exampledata ├── logs └── session # and in /opt/services/flaskapp/src . ├── conf.d ├── python └── static ├── css ├── js └── templates ############################################################################## ############################################################################## ### Docker from DBL for agotool local (Ody) 0. write proper requirements.txt (NOT pip freeze > requirement.txt) pipreqs --force . 1. build images from docker-compse.yml docker-compose build 2. run images in background docker-compose up -d 3. get the container id for the PostgreSQL DB docker ps 4. run psql scripts to populate DB (from host run the psql script in the container) # First time setup docker exec -it agotool_db_1 psql -U postgres -d postgres -f /agotool_data/data/PostgreSQL/create_DBs.psql docker exec -it agotool_db_1 --volume "agotool_agotool_data:/agotool_data" psql -U postgres -d postgres -f ./app/sql/create_DBs.psql # TESTING docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f ./app/postgres/copy_from_file_and_index_DELETE.psql docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f ./app/postgres/drop_and_rename.psql # real DB/not testing docker exec -it agotool_db_1 psql -U postgres -d agotool -f ./app/postgres/copy_from_file_and_index.psql docker exec -it agotool_db_1 psql -U postgres -d agotool -f ./app/postgres/drop_and_rename.psql # test DB docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /Users/dblyon/modules/cpr/agotool/data/PostgreSQL/copy_from_file_and_index_TEST_ODY.psql docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /Users/dblyon/modules/cpr/agotool/data/PostgreSQL/drop_and_rename.psql # populate DB for real docker exec -it agotool_db_1 psql -U postgres -d agotool -f /Users/dblyon/modules/cpr/agotool/data/PostgreSQL/copy_from_file_and_index_ODY.psql docker exec -it agotool_db_1 psql -U postgres -d agotool -f /Users/dblyon/modules/cpr/agotool/data/PostgreSQL/drop_and_rename.psql 5. monthly UPDATES # spin up another instance of agotool_flaskapp image as temporary container that is removed after the commands finish docker run --rm -it --name update --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp python ./app/python/update_manager.py # check out the named volume docker run --rm -it --name checkvolume --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp bash # check out the Postgres DB docker run --rm -it --name checkdb --volume "dbdata:/var/lib/postgresql/data" postgres bash ############################################################################## ##### copy data to named volume # spin up another container that deletes itself after it is done docker run --rm -it --volume /data_from_ody:/mounted_data --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp rsync -avr /mounted_data/data /agotool_data/ docker run --rm -it --volume /agotool_data:/mounted_data --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp bash # as soon as I exit the container will delete itself. Self destruction # volume data persists # alternative to 2 named volumes: # 1 volume with 2 sources and 2 different mount points ############################################################################## ##### crontab on red hat virtual server SHELL=/bin/bash PATH=/sbin:/bin:/usr/sbin:/usr/bin MAILTO=root # For details see man 4 crontabs # Example of job definition: # .---------------- minute (0 - 59) # | .------------- hour (0 - 23) # | | .---------- day of month (1 - 31) # | | | .------- month (1 - 12) OR jan,feb,mar,apr ... # | | | | .---- day of week (0 - 6) (Sunday=0 or 7) OR sun,mon,tue,wed,thu,fri,sat # | | | | | # * * * * * user-name command to be executed # added David Lyon 0 4 1 * * root /var/www/agotool/app/update_DB.sh >> /agotool_data/data/logs/updates_log.txt 2>&1 & #30 * * * * dblyon /home/dblyon/bin/restart_if_down.sh ############################################################################## ### restart if down, deprecated and needs debugging #!/bin/bash if ps aux | grep "python3 runserver.py" > /dev/null then echo "Running" else echo "Stopped" mail -s "aGOtool is restarting" dblyon@gmail.com < /var/www/agotool/logs/update.log cd /var/www/agotool sudo -u agotool python3 runserver.py >> /var/www/agotool/logs/runserver.log 2>&1 & fi ############################################################################## #### backup and restore docker named volume # backup docker run --rm \ --volume [DOCKER_COMPOSE_PREFIX]_[VOLUME_NAME]:/[TEMPORARY_DIRECTORY_TO_STORE_VOLUME_DATA] \ --volume $(pwd):/[TEMPORARY_DIRECTORY_TO_STORE_BACKUP_FILE] \ ubuntu \ tar cvf /[TEMPORARY_DIRECTORY_TO_STORE_BACKUP_FILE]/[BACKUP_FILENAME].tar /[TEMPORARY_DIRECTORY_TO_STORE_VOLUME_DATA] # e.g. docker run --rm --volume agotool_agotool_data:/named_volume_dir --volume $(pwd):/bind_mount_dir ubuntu tar cvf /bind_mount_dir/named_volume_agotool_data_backup_20180209.tar /named_volume_dir docker run --rm --volume agotool_dbdata:/named_volume_dir --volume $(pwd):/bind_mount_dir ubuntu tar cvf /bind_mount_dir/named_volume_dbdata_backup_20180209.tar /named_volume_dir # restore docker run --rm \ --volume [DOCKER_COMPOSE_PREFIX]_[VOLUME_NAME]:/[TEMPORARY_DIRECTORY_STORING_EXTRACTED_BACKUP] \ --volume $(pwd):/[TEMPORARY_DIRECTORY_TO_STORE_BACKUP_FILE] \ ubuntu \ tar xvf /[TEMPORARY_DIRECTORY_TO_STORE_BACKUP_FILE]/[BACKUP_FILENAME].tar -C /[TEMPORARY_DIRECTORY_STORING_EXTRACTED_BACKUP] --strip 1 # e.g. docker run --rm --volume agotool_agotool_data:/named_volume_dir --volume $(pwd):/bind_mount_dir ubuntu tar xvf /bind_mount_dir/named_volume_agotool_data_backup_20180209.tar -C /named_volume_dir --strip 1 docker run --rm --volume agotool_dbdata:/named_volume_dir --volume $(pwd):/bind_mount_dir ubuntu tar xvf /bind_mount_dir/named_volume_dbdata_backup_20180209.tar -C /named_volume_dir --strip 1 ##### beta version on aquarius ### backup docker run --rm --volume agotool_agotool_data:/named_volume_dir --volume $(pwd):/bind_mount_dir ubuntu tar cvf /bind_mount_dir/named_volume_agotool_data_backup_20180903.tar /named_volume_dir docker run --rm --volume agotool_dbdata:/named_volume_dir --volume $(pwd):/bind_mount_dir ubuntu tar cvf /bind_mount_dir/named_volume_dbdata_backup_20180903.tar /named_volume_dir ### copy files rsync -avv -e 'ssh -p 22222' /Users/dblyon/modules/cpr/agotool/named_volume_agotool_data_backup_20180903.tar dblyon@aquarius.meringlab.org://home/dblyon/agotool/data/named_volume_dbdata_backup_20180903.tar rsync -avv -e 'ssh -p 22222' ./named_volume_dbdata_backup_20180903.tar dblyon@aquarius.meringlab.org://home/dblyon/agotool/data/named_volume_dbdata_backup_20180903.tar ### restore on aquarius docker run --rm --volume agotool_agotool_data:/named_volume_dir --volume $(pwd):/bind_mount_dir ubuntu tar xvf /bind_mount_dir/named_volume_agotool_data_backup_20180903.tar -C /named_volume_dir --strip 1 docker run --rm --volume agotool_dbdata:/named_volume_dir --volume $(pwd):/bind_mount_dir ubuntu tar xvf /bind_mount_dir/named_volume_dbdata_backup_20180903.tar -C /named_volume_dir --strip 1 ############################################################################## ### tag and push flaskapp to dockerhub/docker cloud docker tag 3d5c5b99296e dblyon/agotool_flaskapp docker push dblyon/agotool_flaskapp ############################################################################## # ToDo # - push to dockerhub --> done # - set branch docker to master --> done # - create webhooks a la Lars (see below) git commit and push --> github/bitbucket notifies Dockerhub --> Dockerhub builds image --> missing piece: server pulls image and starts it up # Drone IO (Milan is using on this) # Lars automatic updates/webhooks bitbucket/github webhooks on C++ repo Dockerhub build settings trigger URL paste webhook m dockerhub to bitbucket ############################################################################## ### asking Tim to change email address docker run --rm -it --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp bash ############################################################################## ###### @ UZH IMLS # change preload to False in ../app/python/variables.py, PRELOAD = False otherwise flask needs access to Postgres which isn't populated yet # build the images docker-compose build # pushed to image to dockerhub export DOCKER_ID_USER="dblyon" docker login docker push dblyon/python_agotool # commented out the "build" line in order to pull from dockerhub next time instead of building from Dockerfile flaskapp: # build: . # uncomment to build from Dockerfile, else pull from dockerhub image: dblyon/python_agotool:latest # start the containers docker-compose up -d # tmux and download newest resources and create DB files for import docker run --rm -it --name update --user 5009:5009 --volume /mnt/mnemo5/dblyon/agotool:/agotool_data dblyon/python_agotool python ./app/python/update_manager.py # test DB docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /agotool_data/data/PostgreSQL/copy_from_file_and_index_DELETE.psql docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /agotool_data/data/PostgreSQL/drop_and_rename.psql # populate DB for real docker exec -it agotool_db_1 psql -U postgres -d agotool -f /agotool_data/data/PostgreSQL/copy_from_file_and_index.psql docker exec -it agotool_db_1 psql -U postgres -d agotool -f /agotool_data/data/PostgreSQL/drop_and_rename.psql # change "preload" to True, change "skip_slow_downloads" to False, change "debug" to False ############################################################################## # really executed above # scratch below # copy data to named volume (spin up another container that deletes itself after it is done) docker run --rm -it --volume /mnt/mnemo5/dblyon/agotool/data:/mounted_data --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp rsync -avr /mounted_data /agotool_data/ # download newest resources docker run --rm -it --name update --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp python ./app/python/update_manager.py # run containers and create "dbdata" volume docker-compose up -d # change preload to False in ../app/python/variables.py, PRELOAD = False vim /var/www/agotool/app/python/variables.py docker run --rm -it --name work --volume /mnt/mnemo5/dblyon/agotool/data:/mounted_data --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp bash ### up next # test DB docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /agotool_data/data/PostgreSQL/copy_from_file_and_index_DELETE.psql docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /agotool_data/data/PostgreSQL/drop_and_rename.psql docker run --rm -it --name test --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp bash #### DEBUG # change docker default location to store files # check disk usage of default storage loaction sudo du -sh /var/lib/docker 788K /var/lib/docker #### How to store data # Postgres data in named volume "dbdata" - "dbdata:/var/lib/postgresql/data" # Otherwise might be a better solution to not have another named volume like at CPR/SUND # but a bind mount - /mnt/mnemo5/dblyon/agotool/data:/agotool_data docker run --rm -it --name test --volume /mnt/mnemo5/dblyon/agotool:/agotool_data centos bash docker exec -it bash ### RSYNC between Ody and Atlas rsync -av --exclude .git /Users/dblyon/modules/cpr/agotool/ dblyon@imlslnx-atlas.uzh.ch:/mnt/mnemo5/dblyon/agotool rsync -av --exclude .git dblyon@imlslnx-atlas.uzh.ch:/mnt/mnemo5/dblyon/agotool/ /Users/dblyon/modules/cpr/agotool ########################################################################################## ### Tim at SUND to change email address # log into virtual server # change files locally at # There are 2 files which would have to be edited: vim /var/www/agotool/app/static/templates/FAQ.html (1x at the bottom) vim /var/www/agotool/app/static/templates/about.html (2x at the top and in the middle) # stop the docker containers docker-compose down # build the images again, which should incorporate the local changes docker-compose build # start the docker containers again docker-compose up -d --scale flaskapp=2 # check that things are running correctly, this should list 4 containers (2x flaskapp, 1x postgres, 1x nginx) docker ps ########################################################################################## docker run --rm -it --name test --user 5009:5009 --volume /mnt/mnemo5/dblyon/agotool:/agotool_data dblyon/python_agotool bash docker exec -it agotool_flaskapp_1 bash ########################################################################################## ### @ Ody, local version ### file structure locally, dblyon@Ody ...modules/cpr/agotool restapi $ tree -d . ├── app │ ├── conf.d │ ├── python │ │ └── __pycache__ │ └── static │ ├── css │ ├── js │ └── templates └── data ├── Background_Reference_Proteomes ├── PostgreSQL │ ├── downloads │ ├── static │ └── tables │ └── test ├── exampledata ├── logs └── session # set "PRELOAD = False" vim /app/python/variables.py # build the images docker-compose build # run containers docker-compose up -d # copy data to named volume (spin up another container that deletes itself after it is done) docker run --rm -it --volume /Users/dblyon/modules/cpr/agotool:/mounted_data --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp rsync -avr /mounted_data/data /agotool_data/ # delete data if needed, skipped (for later to save disk space) rm -rf /Users/dblyon/modules/cpr/agotool/data # download newest resources, skipped docker run --rm -it --name update --volume "agotool_agotool_data:/agotool_data" agotool_flaskapp python ./app/python/update_manager.py # test DB docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /agotool_data/data/PostgreSQL/copy_from_file_and_index_DELETE.psql docker exec -it agotool_db_1 psql -U postgres -d agotool_test -f /agotool_data/data/PostgreSQL/drop_and_rename.psql # populate DB for real docker exec -it agotool_db_1 psql -U postgres -d agotool -f /agotool_data/data/PostgreSQL/copy_from_file_and_index.psql docker exec -it agotool_db_1 psql -U postgres -d agotool -f /agotool_data/data/PostgreSQL/drop_and_rename.psql # change "preload" to True # change "debug" to False vim /var/www/agotool/app/python/variables.py # run the app with new settings docker-compose up -d ##### Pytest run tests in test_userinput.py # test works but flask crashes "error code 247", restart flask with docker-compose up -d docker exec -it agotool_flaskapp_1 pytest -v ./python/test_userinput.py --> /opt/services/flaskapp/src/env_file ## connect container to network docker run --rm -it --network="agotool_db_nw" --name testing --volume "agotool_agotool_data:/agotool_data" --volume "agotool_dbdata:/dbdata" agotool_flaskapp bash # check DB connection cd app/python python import query c = query.get_cursor_docker(host="db", dbname="agotool", user="postgres", password="USE_YOUR_PASSWORD", port="5432") # pass environmental variables at container creation docker run --rm -it --network="agotool_db_nw" --name testing -e POSTGRES_USER="postgres" -e POSTGRES_PASSWORD="USE_YOUR_PASSWORD" -e POSTGRES_DB="agotool" -e APP_SECRET_KEY="USE_YOUR_SECRET_KEY" --volume "agotool_agotool_data:/agotool_data" --volume "agotool_dbdata:/dbdata" agotool bash # ImportMismatchError from pytest docker run --rm -it --network="agotool_db_nw" --name testing -e POSTGRES_USER="postgres" -e POSTGRES_PASSWORD="USE_YOUR_PASSWORD" -e POSTGRES_DB="agotool" -e APP_SECRET_KEY="USE_YOUR_SECRET_KEY" --volume "agotool_agotool_data:/agotool_data" --volume "agotool_dbdata:/dbdata" agotool pytest -v ./app/python/test_userinput.py --> /opt/services/flaskapp/src/app/env_file docker run --rm -it --network="agotool_db_nw" --name testing -e POSTGRES_USER="postgres" -e POSTGRES_PASSWORD="USE_YOUR_PASSWORD" -e POSTGRES_DB="agotool" -e APP_SECRET_KEY="USE_YOUR_SECRET_KEY" --volume "agotool_agotool_data:/agotool_data" --volume "agotool_dbdata:/dbdata" agotool bash ### connect to PostgreSQL container from host psql --host localhost -U postgres -d agotool ### error 247 docker-compose up --> 3 containers running docker exec -it agotool_flaskapp_1 pytest -v ./python/test_userinput.py --> works but error/crash "agotool_flaskapp_1 exited with code 247" docker-compose up -d --> restart flaskapp docker run --rm -it --network="agotool_db_nw" --name testing -e POSTGRES_USER="postgres" -e POSTGRES_PASSWORD="USE_YOUR_PASSWORD" -e POSTGRES_DB="agotool" -e APP_SECRET_KEY="USE_YOUR_SECRET_KEY" --volume "agotool_agotool_data:/agotool_data" --volume "agotool_dbdata:/dbdata" agotool pytest -v ./app/python/test_userinput.py --> py._path.local.LocalPath.ImportMismatchError: ('conftest', '/opt/services/flaskapp/src/python/conftest.py', local('/opt/services/flaskapp/src/app/python/conftest.py')) ERROR: could not load /opt/services/flaskapp/src/app/python/conftest.py Why is there a dicrepancy in the path: docker exec -it agotool_flaskapp_1 bash --> /opt/services/flaskapp/src/python/test_userinput.py vs docker run --rm -it --network="agotool_db_nw" --name testing -e POSTGRES_USER="postgres" -e POSTGRES_PASSWORD="USE_YOUR_PASSWORD" -e POSTGRES_DB="agotool" -e APP_SECRET_KEY="USE_YOUR_SECRET_KEY" --volume "agotool_agotool_data:/agotool_data" --volume "agotool_dbdata:/dbdata" agotool bash --> /opt/services/flaskapp/src/app/python/test_userinput.py docker run --rm -it --volume "/Users/dblyon/modules/cpr/agotool/app/:/opt/services/flaskapp/src" agotool bash docker run --rm -it --network="agotool_db_nw" --name testing -e POSTGRES_USER="postgres" -e POSTGRES_PASSWORD="USE_YOUR_PASSWORD" -e POSTGRES_DB="agotool" -e APP_SECRET_KEY="USE_YOUR_SECRET_KEY" --volume "/Users/dblyon/modules/cpr/agotool/app/:/opt/services/flaskapp/src" agotool pytest -v ./app/python/test_userinput.py ### PYTEST workaround to flask crashing due to unknown reason # run docker without preloading so the website will not work, but the connection to the DB will be established and therefore # the python container will not crash # set PRELOAD = False # pre-load objects DB connection necessary --> fast load # to load DB docker-compose up # repeatedly call the following, while editing pytest tests (flask dies, but DB stays alive and load is quick) docker exec -it agotool_flaskapp_1 pytest -v ./python/test_userinput.py # Debugging docker exec -it agotool_flaskapp_1 pytest -vx ./python/test_userinput.py --pdb docker exec -it agotool_flaskapp_1 pytest -vx ./python/test_query.py --pdb # Code coverage docker exec -it agotool_flaskapp_1 pytest --cov # failed tests run first docker exec -it agotool_flaskapp_1 pytest -vxff ./python/test_userinput.py ### run pytest with separate container, not needing to turn off 'preload' # atlas docker run -it --net agotool_db_nw --env-file /home/dblyon/agotool/app/env_file -v "/home/dblyon/agotool/data/:/agotool_data" -v "/home/dblyon/agotool/app/:/opt/services/flaskapp/src" agotool_flaskapp:latest pytest -vx ./python/test_query.py --pdb # ody docker run -it --net agotool_db_nw --env-file /Users/dblyon/modules/cpr/agotool/app/env_file -v "/Users/dblyon/modules/cpr/agotool/data/:/agotool_data" -v "/Users/dblyon/modules/cpr/agotool/app/:/opt/services/flaskapp/src" agotool_flaskapp:latest pytest -vx ./python/test_query.py --pdb # copy example data # You can think of a trailing / on a source as meaning "copy the contents of this directory" as opposed to "copy the directory by name" docker run --rm -it --volume /Users/dblyon/modules/cpr/agotool:/mounted_data --volume "agotool_agotool_data:/agotool_data" agotool rsync -avr /mounted_data/data/exampledata/ /agotool_data/data/exampledata #### building working DB for STRING version_ # create DBs docker exec -it postgres psql -U postgres -d postgres -f /agotool_data/PostgreSQL/create_DBs.psql docker run -it --net agotool_db_nw --env-file /Users/dblyon/modules/cpr/agotool/app/env_file -v "/Users/dblyon/modules/cpr/agotool/data/:/agotool_data" -v "/Users/dblyon/modules/cpr/agotool/app/:/opt/services/flaskapp/src" agotool_flaskapp:latest python ./python/create_SQL_tables.py # populate DB for real docker exec -it postgres psql -U postgres -d gostring -f /agotool_data/PostgreSQL/copy_from_file_and_index_STRING.psql docker exec -it postgres psql -U postgres -d gostring -f /agotool_data/PostgreSQL/drop_and_rename_STRING.psql docker exec -it postgres psql -U postgres -d gostring -f /agotool_data/PostgreSQL/temp.psql docker run -it --net agotool_db_nw --env-file /Users/dblyon/modules/cpr/agotool/app/env_file -v "/Users/dblyon/modules/cpr/agotool/data/:/agotool_data" -v "/Users/dblyon/modules/cpr/agotool/app/:/opt/services/flaskapp/src" agotool_flaskapp:latest pytest -vx ./python/test_query.py --pdb docker run -it --net agotool_db_nw --env-file /home/dblyon/agotool/app/env_file -v "/home/dblyon/agotool/data/:/agotool_data" -v "/home/dblyon/agotool/app/:/opt/services/flaskapp/src" agotool_flaskapp:latest bash docker run -it --net agotool_db_nw --env-file /home/dblyon/agotool/app/env_file -v "/home/dblyon/agotool/data/:/agotool_data" -v "/home/dblyon/agotool/app/:/opt/services/flaskapp/src" agotool_flaskapp:latest python docker run -it --net agotool_db_nw --env-file /home/dblyon/agotool/app/env_file -v "/home/dblyon/agotool/data/:/agotool_data" -v "/home/dblyon/agotool/app/:/opt/services/flaskapp/src" agotool_flaskapp:latest python string_like_output.py # change "preload" to True, change "skip_slow_downloads" to False, change "debug" to False vim /var/www/agotool/app/python/variables.py --> tadaaa it should work now ### scale a service like the flask-app docker-compose up -d --scale flaskapp=2 ############################################################################## # http://agotool-api.meringlab.org/ # aquarius # http://agotool.meringlab.org/ # atlas # try a query from the CLI curl -X POST "localhost:5912/api?enrichment_method=genome&taxid=9606" -d '{"foreground": "9606.ENSP00000251595%0d9606.ENSP00000322421%0d9606.ENSP00000333994"}' curl -X POST "localhost:5912/api?enrichment_method=genome&taxid=9606" -d "9606.ENSP00000259606.ENSP00000322421" /Users/dblyon/anaconda3/lib/gcc/x86_64-apple-darwin11.4.2/4.8.5/include-fixed/limits.h /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/include/c++/v1 ############################################################################## ### flaskapp.conf /nginx backup for single flaskapp container/service server { listen 80; server_name localhost; large_client_header_buffers 4 8m; client_header_buffer_size 1k; client_body_buffer_size 10M; client_max_body_size 10M; location / { proxy_set_header Host $host; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://flaskapp:5912; proxy_connect_timeout 6000; proxy_send_timeout 6000; proxy_read_timeout 6000; send_timeout 6000; } } ############################################################################## ### flaskapp.conf / nginx for multiple containers upstream flaskapp { server flaskapp_1:5912; server flaskapp_2:5912; } server { listen 80; server_name localhost; large_client_header_buffers 4 8m; client_header_buffer_size 1k; client_body_buffer_size 10M; client_max_body_size 10M; # location ~ \.(css|js|ico|png)$ { # root /var/www; # expires 2h; # access_log off; # } location ^~ /static { root /var/www; expires 2h; access_log off; } location / { proxy_set_header Host $host; proxy_set_header Host $http_host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://flaskapp; proxy_connect_timeout 6000; proxy_send_timeout 6000; proxy_read_timeout 6000; send_timeout 6000; proxy_max_temp_file_size 0; } } rsync -avP --recursive --files-from=./files_for_DB_STRINGv11.txt . /mnt/mnemo5/dblyon/agotool/data/PostgreSQL/tables/bak_v11 rsync -avP --recursive --files-from=./files_for_DB_STRINGv11.txt dblyon@imlslnx-atlas.uzh.ch:/home/dblyon/agotool/data/PostgreSQL/tables ./ rsync -avP --recursive --files-from=./files_for_DB_STRINGv11.txt . dblyon@san.embl.de:/home/dblyon/agotool/data/PostgreSQL/tables/ docker-compose up -d --no-deps --build service_name docker build -t flaskapp_alpine:latest -f ./Dockerfile_alpine . conda remove --name agotool --all conda env create -n agotool -f conda_agotool.yml python setup.py build_ext --inplace conda_agotool.yml: - clang - gcc - libgcc ############################################################################################################## # Protein_2_Function_and_Score_DOID_GO_BTO.txt integrated_protein2function.tsv # Protein 2 Function and Score relationships for etypes -22, -25, and -26 (GO cellular component, BTO tissues, and DOID diseases) 6239.C30G4.7 {{"GO:0043226",0.875},{"GO:0043227",0.875},{"GO:0043231",0.875},{"GO:0044424",2.96924}, ... , {"GO:0005737",2.742276},{"GO:0005777",0.703125}} -22 10116.ENSRNOP00000049139 {{"GO:0005623",2.927737},{"GO:0044424",2.403304},{"GO:0044425",3},{"GO:0031224",3}, ... ,{"GO:0043232",0.375}} -22 - remove anything on blacklist (all_hidden.tsv) already happend while creating Functions_table_DOID_BTO (and all terms not present therein will be filtered out) - omit GO-CC (etype -22) Variant A.) using a hard cutoff of e.g. >= 3 - create Protein_2_Function_table_DOID and - create Protein_2_Function_table_BTO and add them to Protein_2_Function_table_STRING.txt Variant B.) use continuous scores - translate function_name to function_enum - create Protein_2_FunctionEnum_and_Score_table_STRING.txt (BTO and DOID, etype -25 and -26) - create Taxid_2_FunctionCountArray_BTO_DOID.txt (add values to array using a function instead of merging tables) # Protein_2_FunctionEnum_and_Score_table_STRING.txt | 10116.ENSRNOP00000049139 | {{{0,2.927737},{3,2.403304},{4,3},{666,3}, ... ,{3000000,0.375}} | # sum scores and scale to sum up to foreground_n # --> scaling_factor = sum(scores) / foreground_n # score * scaling_factor # create precomputed counts per genome analogous to Taxid_2_FunctionCountArray_table_STRING.txt # Taxid_2_FunctionCountArray_2_merge_BTO_DOID.txt | 9606 | 19566 | {{{0,3},{3,2},{4,3},{666,3}, ... ,{3000000,1}} | # sum scores and scale to sum up to background_n # --> scaling_factor = sum(scores) / background_n # score * scaling_factor --> and round to closest integer # Functions_table_DOID_BTO.txt integrated_function2description.tsv # GO, BTO, and DOID function descriptions - add hierarchical level, year placeholder - merge with Functions_table - add things to Lineage table -26 DOID:0050483 DOID:0050483 -26 DOID:0050662 Bestrophinopathy -26 DOID:0060317 Lung abscess -25 BTO:0003000 EB-1 cell -22 GO:0099012 Neuronal dense core vesicle membrane Protein_2_Function_table_PMID_STS download_Function_2_Description_PMID Function_table_PMID_temp Protein_2_FunctionEnum_and_Score_table_STRING Functions_table_DOID_BTO Taxid_2_FunctionCountArray_2_merge_BTO_DOID download_Interpro_descriptions ################################################################################ ##### installing/restarting aGOtool ### get the code to run the python flask app from github # clone git repo (master branch is STRING v11) git clone https://github.com/dblyon/agotool.git git checkout STRING_v11 cd ./agotool/app ### install anaconda (follow online instructions) # create an enviroment using requirements in yml file conda env create -n agotool -f conda_agotool.yml # get rid of the conda prompt conda config --show | grep changeps1 conda config --set changeps1 False ### push the data (flat files) from e.g. Atlas/Gaia to San/Pisces using zipped file rsync -avP /mnt/mnemo5/dblyon/agotool/data/PostgreSQL/tables/bak_v11.3/STRING_v11.3_flat_files.zip dblyon@san.embl.de:/home/dblyon/agotool/data/PostgreSQL/tables/ # or alternatively rsync the txt files cd /mnt/mnemo5/dblyon/agotool/data/PostgreSQL/tables/bak_v11.3 rsync -avP --recursive --files-from=./files_for_DB_STRINGv11.txt . dblyon@san.embl.de:/home/dblyon/agotool/data/PostgreSQL/tables/ # cd to agotool/data/PostgreSQL/tables/ and unzip files unzip STRING_v11.3_flat_files.zip ### activate the environment (or use absolute path of python bin) conda activate agotool # find and select absolute path for python environment (use absolute path of proper env and add "/bin/python") conda info -e ### compile/build the Cythonized extension module cd agotool/app/python # compile run_cythonized.*.so /mnt/mnemo5/dblyon/install/anaconda3/envs/agotool/bin/python setup.py build_ext --inplace ### start the python flask server cd ./agotool/app nohup /mnt/mnemo5/dblyon/install/anaconda3/envs/agotool/bin/python runserver.py &>/dev/null & ################################################################################ #### 20190618 #### building working DB for UniProt aGOtool version # create DBs docker exec -it postgres12 psql -U postgres -d agotool -f /agotool_data/PostgreSQL/create_DBs.psql docker exec -it postgres12 psql -U postgres -d agotool -f /agotool_data/PostgreSQL/copy_from_file_and_index.psql docker exec -it postgres12 psql -U postgres -d agotool -f /agotool_data/PostgreSQL/drop_and_rename.psql # or cd ~/modules/cpr/agotool/data/PostgreSQL psql -d agotool -f copy_from_file_and_index_ODY.psql psql -d agotool -f drop_and_rename.psql # on Aquarius cd /home/dblyon/agotool/data/PostgreSQL psql -d agotool -f copy_from_file_and_index.psql ### Error using new PostgreSQL version: remove PostgreSQL data directory, since switching from v10 to v11 results in problems prune the system with "remove_volumes" in ""~/scripts/docker_volumes.sh " """ removecontainers() { docker stop $(docker ps -aq) docker rm $(docker ps -aq) } remove_volumes() { removecontainers docker network prune -f docker rmi -f $(docker images --filter dangling=true -qa) docker volume rm $(docker volume ls --filter dangling=true -q) docker rmi -f $(docker images -qa) } """ ### files for rsync transfer from file list # from Atlas to Ody, on Ody rsync -avP --update --files-from=/Users/dblyon/modules/cpr/agotool/app/rsync_files_UniProtVersion.txt dblyon@imlslnx-atlas.uzh.ch:/mnt/mnemo5/dblyon/agotool/data/PostgreSQL/tables . # from Atlas to Aquarius, on Atlas rsync -avP --update --files-from=/mnt/mnemo5/dblyon/agotool/app/rsync_files_UniProtVersion.txt . dblyon@aquarius.meringlab.org:/home/dblyon/agotool/data/PostgreSQL/tables # from Ody to Aquarius, on Ody rsync -avP --update --files-from=/Users/dblyon/modules/cpr/agotool/app/rsync_files_UniProtVersion.txt . dblyon@aquarius.meringlab.org:/home/dblyon/agotool/data/PostgreSQL/tables ### uWSGI configuration: agotool/app/uwsgi_config.ini start the app with: uwsgi uwsgi_config.ini