diff --git a/.gitignore b/.gitignore index dd37a27..27eb0a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ data .env -conf \ No newline at end of file +conf +multidocker/conf diff --git a/Dockerfile b/Dockerfile index 4b1118d..c980dfd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,8 +26,23 @@ RUN apt-get update && DEBIAN_FRONTEND="noninteractive" apt-get install -y \ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - -RUN apt-get install -y nodejs +## begin node + +## old node install commands + +# RUN curl -sL https://deb.nodesource.com/setup_14.x | bash - +# RUN apt-get install -y nodejs + +## new node install commands + +ARG node_version=v18.20.2 +RUN cd /opt \ + && curl -LO https://nodejs.org/dist/${node_version}/node-${node_version}-linux-x64.tar.xz \ + && tar xJf node-${node_version}-linux-x64.tar.xz \ + && rm node-${node_version}-linux-x64.tar.xz +ENV PATH=/opt/node-${node_version}-linux-x64/bin:${PATH} + +## end node WORKDIR / @@ -49,6 +64,9 @@ COPY dataorig /dataorig/ # Clone repos WORKDIR / #RUN git clone https://github.com/stt-datacore/website.git #- run script should do this + +# experimental and easy to revert + RUN git clone https://github.com/stt-datacore/cpp-image-analysis.git RUN git clone https://github.com/stt-datacore/bot.git RUN git clone https://github.com/stt-datacore/asset-server.git @@ -95,6 +113,7 @@ RUN make -j$(nproc) #RUN mv /asset-server/out/* /dataorig/asset-server/ #RUN rm -rf /asset-server/out #RUN ln -s /data/asset-server /asset-server/out - +RUN echo Node Version +RUN node --version COPY ./runme.sh /runme.sh ENTRYPOINT /runme.sh diff --git a/multidocker/.env.defaults b/multidocker/.env.defaults index 539d0f3..4ae4ca2 100644 --- a/multidocker/.env.defaults +++ b/multidocker/.env.defaults @@ -1,9 +1,36 @@ -NODE_VERSION=16 +NODE_VERSION=18 ALPINE_VERSION=3.17 + WEBSITE_BRANCH=master SERVER_BRANCH=master ASSET_BRANCH=master BOT_BRANCH=master -BUILD_TYPE=development +CPP_BRANCH=master + NGINX_CONF=development.conf +BUILD_TYPE=development + +# PM2 +PM2_INFO_URL= +PM2_ERROR_URL= + +# Shared Assets +# ASSETS_DIR=/app/asset-server/build/out +# SQLITE_DB=/app/site-server/static/profiles.sqlite + +# MongoDB Environment +MONGO_INITDB_DATABASE=user +MONGO_INITDB_ROOT_USERNAME=admin +MONGO_INITDB_ROOT_PASSWORD=secret +DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db +MONGO_CONN_STRING=mongodb://admin:secret@datacoredb +MONGO_DB_NAME=datacore +MONGO_PROFILE_COLLECTION=playerProfiles +MONGO_TRACKED_VOYAGES_COLLECTION=trackedVoyages +MONGO_TRACKED_ASSIGNMENTS_COLLECTION=trackedAssignments +MONGO_TELEMETRY_VOYAGE_COLLECTION=telemetry +MONGO_FBB_SOLVES_COLLECTION=solves +MONGO_FBB_TRIALS_COLLECTION=trials +MONGO_FBB_COLLECTION=bossBattles +MONGO_DISCORD_USERS_COLLECTION=discordUsers \ No newline at end of file diff --git a/multidocker/conf-defaults/bot.env b/multidocker/conf-defaults/bot.env new file mode 100644 index 0000000..3b62071 --- /dev/null +++ b/multidocker/conf-defaults/bot.env @@ -0,0 +1,18 @@ + +# Bot +BOT_TOKEN= +DATACORE_URL=https://datacore.app/ +ASSETS_URL=https://assets.datacore.app/ +NODE_ENV=production + +IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000 + +PROFILE_DATA_PATH=/profiles/ +DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db +LOG_PATH=/data/logs/ +DC_DATA_PATH=/sitebuild/structured +CONFIG_PATH=/data/bot_config.json + +# ImgFlip +IMGFLIP_PASSWORD= +IMGFLIP_USERNAME= diff --git a/multidocker/conf-defaults/cpp-image-analysis.env b/multidocker/conf-defaults/cpp-image-analysis.env new file mode 100644 index 0000000..06dcc64 --- /dev/null +++ b/multidocker/conf-defaults/cpp-image-analysis.env @@ -0,0 +1,3 @@ +CPP_TRAIN_PATH=/traindata/train/ +CPP_DATA_PATH=/traindata/data/ + diff --git a/multidocker/conf-defaults/rsync.env b/multidocker/conf-defaults/rsync.env new file mode 100644 index 0000000..1b185d5 --- /dev/null +++ b/multidocker/conf-defaults/rsync.env @@ -0,0 +1 @@ +RSYNC_PASSWORD=insecure \ No newline at end of file diff --git a/multidocker/conf-defaults/site-server.env b/multidocker/conf-defaults/site-server.env new file mode 100644 index 0000000..ce5e9cc --- /dev/null +++ b/multidocker/conf-defaults/site-server.env @@ -0,0 +1,7 @@ +# Site Server Environment +STT_BOT_USERNAME= +STT_BOT_PASSWORD= +JWT_SECRET= +CORS_ORIGIN=https://datacore.app/ +PROFILE_DATA_PATH=/profiles +LOG_PATH=/data/logs diff --git a/multidocker/conf-defaults/website.env b/multidocker/conf-defaults/website.env new file mode 100644 index 0000000..749a347 --- /dev/null +++ b/multidocker/conf-defaults/website.env @@ -0,0 +1,3 @@ +# Website Environment +GATSBY_DATACORE_URL=/ +GATSBY_ASSETS_URL=/assets/ diff --git a/multidocker/conf.defaults/bot.env b/multidocker/conf.defaults/bot.env new file mode 100644 index 0000000..45f73d0 --- /dev/null +++ b/multidocker/conf.defaults/bot.env @@ -0,0 +1,22 @@ + +# Bot +BOT_TOKEN= +DATACORE_URL=https://datacore.app/ +ASSETS_URL=https://assets.datacore.app/ +DEFAULT_GUILD= + +# Seperate configuration for bot, because certain dev scenarios +# require that this be different +NODE_ENV=development + +IMAGE_ANALYSIS_URL=http://DCImageAnalysis:5000 + +PROFILE_DATA_PATH=/profiles/ +DB_CONNECTION_STRING=sqlite:/sqlite/datacore.db +LOG_PATH=/data/logs/ +DC_DATA_PATH=/sitebuild/structured +CONFIG_PATH=/data/bot_config.json + +# ImgFlip +IMGFLIP_PASSWORD= +IMGFLIP_USERNAME= diff --git a/multidocker/conf.defaults/cpp-image-analysis.env b/multidocker/conf.defaults/cpp-image-analysis.env new file mode 100644 index 0000000..06dcc64 --- /dev/null +++ b/multidocker/conf.defaults/cpp-image-analysis.env @@ -0,0 +1,3 @@ +CPP_TRAIN_PATH=/traindata/train/ +CPP_DATA_PATH=/traindata/data/ + diff --git a/multidocker/conf.defaults/site-server.env b/multidocker/conf.defaults/site-server.env index 68c054f..ce5e9cc 100644 --- a/multidocker/conf.defaults/site-server.env +++ b/multidocker/conf.defaults/site-server.env @@ -1,5 +1,7 @@ +# Site Server Environment STT_BOT_USERNAME= STT_BOT_PASSWORD= JWT_SECRET= -DB_CONNECTION_STRING=sqlite://database -CORS_ORIGIN=https://datacore.app/ \ No newline at end of file +CORS_ORIGIN=https://datacore.app/ +PROFILE_DATA_PATH=/profiles +LOG_PATH=/data/logs diff --git a/multidocker/conf.defaults/website.env b/multidocker/conf.defaults/website.env index 66914ad..749a347 100644 --- a/multidocker/conf.defaults/website.env +++ b/multidocker/conf.defaults/website.env @@ -1,3 +1,3 @@ +# Website Environment GATSBY_DATACORE_URL=/ -#GATSBY_ASSETS_URL=https://assets.datacore.app/ -GATSBY_ASSETS_URL=/assets/ \ No newline at end of file +GATSBY_ASSETS_URL=/assets/ diff --git a/multidocker/contexts/bot/Dockerfile b/multidocker/contexts/bot/Dockerfile index 638c778..8785bf5 100644 --- a/multidocker/contexts/bot/Dockerfile +++ b/multidocker/contexts/bot/Dockerfile @@ -11,4 +11,4 @@ COPY bot_config.json /data/ CMD npm run start FROM development as production -CMD pm2-runtime --name server start npm -- start \ No newline at end of file +CMD pm2-runtime --name bot start npm -- start \ No newline at end of file diff --git a/multidocker/contexts/cpp-image-analysis/Dockerfile b/multidocker/contexts/cpp-image-analysis/Dockerfile index 21867f9..5b6adcf 100644 --- a/multidocker/contexts/cpp-image-analysis/Dockerfile +++ b/multidocker/contexts/cpp-image-analysis/Dockerfile @@ -6,7 +6,14 @@ ENV LANG=C.UTF-8 WORKDIR /usr/src/dcimageanalysis COPY --from=source . . +WORKDIR /traindata +RUN mkdir train +RUN cp /usr/src/dcimageanalysis/data . -r + +WORKDIR /usr/src/dcimageanalysis + RUN apk add --no-cache --virtual .build \ + curl \ build-base \ pkgconfig \ cmake \ @@ -50,4 +57,5 @@ WORKDIR /usr/src/dcimageanalysis/build CMD ./imserver FROM development as production -CMD pm2-runtime start "./imserver --jsonpath=/website/public_web/structured/ --trainpath=/data/train/ --datapath=/cpp-image-analysis/data/ --asseturl=https://assets.datacore.app/" --name "imageAnalysis" + +CMD pm2-runtime start "./imserver --jsonpath=/sitebuild/structured/ --trainpath=$CPP_TRAIN_PATH --datapath=$CPP_DATA_PATH --asseturl=$ASSETS_URL" --name "imageAnalysis" diff --git a/multidocker/contexts/gittower/Dockerfile b/multidocker/contexts/gittower/Dockerfile index 7851bbf..8577acb 100644 --- a/multidocker/contexts/gittower/Dockerfile +++ b/multidocker/contexts/gittower/Dockerfile @@ -1,7 +1,8 @@ ARG ALPINE_VERSION -FROM alpine:${ALPINE_VERSION} -RUN apk add docker git bash -RUN echo */30 * * * * ./runjobs > crontab +# FROM alpine:${ALPINE_VERSION} +FROM alpine:latest +RUN apk add docker git bash docker-compose +RUN echo "*/5 * * * * /app/runjobs" > crontab RUN crontab ./crontab RUN mkdir /app WORKDIR /app diff --git a/multidocker/contexts/gittower/gittower b/multidocker/contexts/gittower/gittower index d91098c..eef25fe 100755 --- a/multidocker/contexts/gittower/gittower +++ b/multidocker/contexts/gittower/gittower @@ -1,3 +1,4 @@ #!/bin/bash ./setup-jobs +echo gittower waiting ... crond -f -l 8 \ No newline at end of file diff --git a/multidocker/contexts/gittower/runjobs b/multidocker/contexts/gittower/runjobs index d392fd2..620cce9 100755 --- a/multidocker/contexts/gittower/runjobs +++ b/multidocker/contexts/gittower/runjobs @@ -13,3 +13,4 @@ do git fetch git status|grep "is behind" && reload ${dir##*/} done +echo gittower waiting ... diff --git a/multidocker/contexts/gittower/setup-jobs b/multidocker/contexts/gittower/setup-jobs index 8c3a2bd..0f41f04 100755 --- a/multidocker/contexts/gittower/setup-jobs +++ b/multidocker/contexts/gittower/setup-jobs @@ -4,4 +4,5 @@ BASE_URL=https://github.com/stt-datacore/ ./watch website ${BASE_URL}website.git ${WEBSITE_BRANCH} ./watch site-server ${BASE_URL}site-server.git ${SERVER_BRANCH} -./watch asset-server ${BASE_URL}asset-server.git ${ASSET_BRANCH} \ No newline at end of file +./watch asset-server ${BASE_URL}asset-server.git ${ASSET_BRANCH} +./watch bot ${BASE_URL}bot.git ${BOT_BRANCH} diff --git a/multidocker/contexts/nginx/local.conf b/multidocker/contexts/nginx/local.conf new file mode 100644 index 0000000..419ccb3 --- /dev/null +++ b/multidocker/contexts/nginx/local.conf @@ -0,0 +1,31 @@ +server { + listen 80; + client_max_body_size 100M; + #server_name datacore.local; + + location / { + root /sitebuild; + expires 1h; + } + + location /assets/ { + root /assets/; + access_log off; + expires max; + } + + location /assets/data/ { + root /assets/data/; + expires 1h; + } + + location /profiles/ { + root /profiles/; + expires 1h; + } + + location /api/ { + proxy_pass http://DCSiteServer:4420; + expires -1; + } +} \ No newline at end of file diff --git a/multidocker/contexts/pm2-discord/Dockerfile b/multidocker/contexts/pm2-discord/Dockerfile index 2dd4978..dee34e6 100644 --- a/multidocker/contexts/pm2-discord/Dockerfile +++ b/multidocker/contexts/pm2-discord/Dockerfile @@ -1,7 +1,5 @@ ARG ALPINE_VERSION ARG NODE_VERSION -ARG PM2_ERROR_URL -ARG PM2_INFO_URL FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} RUN npm install pm2 -g @@ -20,5 +18,7 @@ RUN pm2 set pm2-discord-webhook:stop true RUN pm2 set pm2-discord-webhook:restart true # Set destinations for webhooks (URLs sourced from env) -RUN pm2 set pm2-discord-webhook:webhook_url_logs ${PM2_INFO_URL} -RUN pm2 set pm2-discord-webhook:webhook_url_errors ${PM2_ERROR_URL} \ No newline at end of file +ARG PM2_INFO_URL +RUN pm2 set pm2-discord-webhook:webhook_url_logs $PM2_INFO_URL +ARG PM2_ERROR_URL +RUN pm2 set pm2-discord-webhook:webhook_url_errors $PM2_ERROR_URL \ No newline at end of file diff --git a/multidocker/contexts/site-server/Dockerfile b/multidocker/contexts/site-server/Dockerfile index 7cdf0f0..b4b72b9 100644 --- a/multidocker/contexts/site-server/Dockerfile +++ b/multidocker/contexts/site-server/Dockerfile @@ -2,7 +2,7 @@ FROM datacore/pm2-discord as development RUN apk add python3 build-base sqlite RUN mkdir /app WORKDIR /app -COPY --from=source package.json package-lock.json . +COPY --from=source package.json package-lock.json ./ RUN npm install COPY --from=source . . RUN npm run build diff --git a/multidocker/contexts/website/Dockerfile b/multidocker/contexts/website/Dockerfile index 4609668..e76432d 100644 --- a/multidocker/contexts/website/Dockerfile +++ b/multidocker/contexts/website/Dockerfile @@ -1,12 +1,14 @@ ARG NODE_VERSION ARG ALPINE_VERSION -FROM node:${NODE_VERSION}-alpine${ALPINE_VERSION} +FROM node:$NODE_VERSION-alpine$ALPINE_VERSION RUN apk add build-base python3 git rsync WORKDIR /app # Sentry requires the github sha RUN git clone https://github.com/stt-datacore/website.git .git --bare --depth 1 COPY --from=source package.json yarn.lock ./ +RUN yarn set version 3.6.3 RUN yarn install RUN npx update-browserslist-db@latest COPY --from=source . . -CMD yarn run build && rsync -ru --delete-after /app/public/ /sitebuild/ && sleep infinity \ No newline at end of file + +CMD yarn run build && rsync -ru --delete-after /app/public/ /sitebuild/ diff --git a/multidocker/datacore.db b/multidocker/datacore.db deleted file mode 100644 index 919c950..0000000 Binary files a/multidocker/datacore.db and /dev/null differ diff --git a/multidocker/docker-compose.yml b/multidocker/docker-compose.yml index c789859..c1cbc83 100644 --- a/multidocker/docker-compose.yml +++ b/multidocker/docker-compose.yml @@ -1,4 +1,4 @@ -version: '3.8' +version: '3' volumes: sitebuild: assets: @@ -7,12 +7,17 @@ volumes: nginx-logs: site-logs: bot-logs: + datacore-mongo: + sqlite: services: gittower: image: datacore/gittower build: context: contexts/gittower + args: + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} deploy: restart_policy: condition: on-failure @@ -29,25 +34,29 @@ services: build: context: contexts/pm2-discord args: - ALPINE_VERSION: - NODE_VERSION: - PM2_ERROR_URL: - PM2_INFO_URL: - + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} + PM2_ERROR_URL: ${PM2_ERROR_URL} + PM2_INFO_URL: ${PM2_INFO_URL} + env_file: + - .env + profiles: ["all", "fullstack", "site-server", "website", "bot"] + website: - image: datacore/website build: context: contexts/website additional_contexts: source: https://github.com/stt-datacore/website.git#${WEBSITE_BRANCH} args: - ALPINE_VERSION: - NODE_VERSION: + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} env_file: + - .env - ./conf/website.env deploy: restart_policy: condition: on-failure + container_name: DCWebsite volumes: - sitebuild:/sitebuild:rw @@ -60,8 +69,8 @@ services: additional_contexts: source: https://github.com/stt-datacore/asset-server.git#${ASSET_BRANCH} args: - ALPINE_VERSION: - NODE_VERSION: + ALPINE_VERSION: ${ALPINE_VERSION} + NODE_VERSION: ${NODE_VERSION} container_name: DCAssetServer deploy: @@ -70,8 +79,19 @@ services: volumes: - assets:/assets:rw environment: - OUT_PATH: /assets/ - profiles: [ "all", "asset-server", "fullstack"] + - OUT_PATH=/assets/ + profiles: [ "all", "asset-server", "website", "fullstack" ] + + datacoredb: + image: mongo:4.4.6 + container_name: datacoredb + volumes: + - datacore-mongo:/data/db + env_file: + - .env + ports: + - '27017-27019:27017-27019' + profiles: ["all", "site-server", "bot", "fullstack"] site-server: image: datacore/site-server @@ -91,13 +111,15 @@ services: - "4420:4420" volumes: - site-logs:/data/logs:rw - - profiles:/data/profiles:rw" - - ./datacore.db:/database + - profiles:/profiles:rw + - sqlite:/sqlite env_file: - - ./conf/site-server.env + - .env + - ./conf/site-server.env depends_on: - pm2-discord - profiles: ["all", "site-server", "fullstack"] + - datacoredb + profiles: ["all", "site-server", "fullstack", "bot"] nginx: image: nginx:alpine${ALPINE_VERSION} @@ -111,10 +133,13 @@ services: volumes: - sitebuild:/sitebuild:ro - assets:/assets:ro - - profiles:/profiles/profiles:ro - - ./contexts/nginx/${NGINX_CONF}:/etc/nginx/conf.d/default.conf:rw + - profiles:/profiles:ro - nginx-logs:/var/log/nginx:rw - profiles: [ "all", "website", "asset-server", "site-server", "fullstack" ] + - ${NGINX_CONF}:/etc/nginx/conf.d/default.conf:rw + depends_on: + - site-server + - datacoredb + profiles: [ "all", "website", "asset-server", "site-server", "fullstack", "bot" ] node_exporter: image: prom/node-exporter @@ -161,9 +186,9 @@ services: ports: - "1337:22" volumes: - - sitebuild:/data/sitebuild:rw - - assets:/data/assets:rw - - profiles:/data/profiles:rw + - sitebuild:/sitebuild:rw + - assets:/data/assets:ro + - profiles:/profiles:rw - traindata:/data/traindata:rw - nginx-logs:/logs/nginx:ro - site-logs:/logs/site-server:ro @@ -176,21 +201,26 @@ services: build: context: contexts/cpp-image-analysis additional_contexts: - source: https://github.com/stt-datacore/cpp-image-analysis.git + source: https://github.com/stt-datacore/cpp-image-analysis.git#${CPP_BRANCH} args: ALPINE_VERSION: container_name: DCImageAnalysis deploy: restart_policy: condition: on-failure + env_file: + - .env + - ./conf/bot.env + - ./conf/cpp-image-analysis.env ports: - - "5000:5000" + - 5000:5000 volumes: - sitebuild:/sitebuild:ro - - ./contexts/cvdata:/data:ro - traindata:/traindata:rw depends_on: - pm2-discord + - nginx + - website profiles: [ "all", "bot" ] bot: @@ -202,22 +232,20 @@ services: target: ${BUILD_TYPE} args: NODE_VERSION: - container_name: DCBot + container_name: DCBot deploy: restart_policy: condition: on-failure volumes: - - profiles:/data/profiles:rw - sitebuild:/sitebuild:ro + - assets:/data/assets:ro - bot-logs:/data/logs:rw - - assets:/assets:ro - environment: - PROFILE_DATA_PATH: /data/profiles - LOG_PATH: /data/logs/ - DC_DATA_PATH: /sitebuild/structured - CONFIG_PATH: /data/bot_config.json + - profiles:/profiles:rw + - sqlite:/sqlite env_file: + - .env - ./conf/bot.env depends_on: - cpp-image-analysis + - datacoredb profiles: ["all", "bot"] \ No newline at end of file