From cf7ede5495d1349c3e5a9d2830666d18e529d65c Mon Sep 17 00:00:00 2001 From: agres Date: Wed, 19 Mar 2025 21:05:33 +0100 Subject: [PATCH 1/8] Fix startup --- startup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/startup.sh b/startup.sh index 5fb7fd1..7f269f3 100755 --- a/startup.sh +++ b/startup.sh @@ -11,4 +11,4 @@ else exit 1 fi -.venv/bin/python3 src/scraper.py +.venv/bin/python3 src/runtime.py From 2d4e9516931a85bd57b5949eeca91238bb3dccd5 Mon Sep 17 00:00:00 2001 From: Chris Kiriakou Date: Fri, 21 Mar 2025 18:54:07 +0100 Subject: [PATCH 2/8] Move Dockerfile & startup script to sperate directory --- Dockerfile | 24 ------------------------ startup.sh | 14 -------------- 2 files changed, 38 deletions(-) delete mode 100644 Dockerfile delete mode 100755 startup.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 45f0f04..0000000 --- a/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM alpine:latest - -WORKDIR /root - -RUN apk update && \ - apk add --no-cache \ - openssh \ - python3 \ - py3-pip \ - sqlite - -EXPOSE 22 - -RUN mkdir /root/src - -COPY ./startup.sh /root -COPY ./requirements.txt /root -COPY ./src/ /root/src/ - -RUN ls -la - -VOLUME /root - -ENTRYPOINT ["/bin/sh", "/root/startup.sh"] diff --git a/startup.sh b/startup.sh deleted file mode 100755 index 7f269f3..0000000 --- a/startup.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# -# Starup the predictify scraper - -if test -f ./requirements.txt -then - python3 -m venv .venv - .venv/bin/pip install -r ./requirements.txt -else - printf "Missing requirements file! aborting...\n" - exit 1 -fi - -.venv/bin/python3 src/runtime.py From 38e0c69beac5c522fd63feba92767d8fdbaa75d5 Mon Sep 17 00:00:00 2001 From: Chris Kiriakou Date: Fri, 21 Mar 2025 18:57:13 +0100 Subject: [PATCH 3/8] Move `.env` file to config dir, change db path for improved project structure * Change path of db and token to `data` which helps to sperate source code from data files like `tokens.json` & `spotify_scraped.db` * Change path of `.env` `src/env/` to `config/` for the same reason as mentioned above * Added the newly created files to `.gitigore` accordingly --- src/auth.py | 4 ++-- src/env/.env.example | 3 --- src/scraper.py | 2 +- 3 files changed, 3 insertions(+), 6 deletions(-) delete mode 100644 src/env/.env.example diff --git a/src/auth.py b/src/auth.py index 7f714da..57785aa 100644 --- a/src/auth.py +++ b/src/auth.py @@ -9,7 +9,7 @@ from urllib.parse import parse_qs, urlencode, urlparse import dotenv import requests -TOKEN_FILE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'env', 'tokens.json') +TOKEN_FILE_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../data', 'tokens.json') def simple_authenticate(grant_type: str = "client_credentials") -> str: @@ -102,7 +102,7 @@ def _read_env_file() -> tuple: :return: tuple """ current_dir = os.path.dirname(os.path.abspath(__file__)) - dotenv_folder_path = os.path.join(current_dir, 'env') + dotenv_folder_path = os.path.join(current_dir, '../config') dotenv_path = os.path.join(dotenv_folder_path, '.env') contents = dotenv.dotenv_values(dotenv_path=dotenv_path) spotify_client_id = contents['SPOTIFY_CLIENT_ID'] diff --git a/src/env/.env.example b/src/env/.env.example deleted file mode 100644 index 545e283..0000000 --- a/src/env/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -SPOTIFY_CLIENT_ID=your_token_here -SPOTIFY_CLIENT_SECRET=your_token_here -SPOTIFY_REDIRECT_URI=http://localhost:8888/callback diff --git a/src/scraper.py b/src/scraper.py index 794b444..3f57274 100644 --- a/src/scraper.py +++ b/src/scraper.py @@ -3,7 +3,7 @@ import requests from auth import authenticate, simple_authenticate from database_handler import Database, Table -db = Database('spotify_scraped.db') +db = Database('./data/spotify_scraped.db') def scraping(): From 3922c7640eba8cf69ef2c2a22285dbc7cdfd7eda Mon Sep 17 00:00:00 2001 From: Chris Kiriakou Date: Fri, 21 Mar 2025 19:05:00 +0100 Subject: [PATCH 4/8] Add documentation for docker and small visual improvements --- README.md | 38 +++++++++++++++++++++++++++++++++----- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f0a6c4a..8f3b60b 100644 --- a/README.md +++ b/README.md @@ -11,15 +11,43 @@ A Data analysis tool to scrape your Spotify History usage and let a ML-Model pre ## Usable possible APIs -Recently Played Tracks: /me/player/recently-played [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-recently-played) +Recently Played Tracks: `/me/player/recently-played` [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-recently-played) -Get Track: /tracks/{id} [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-track) +Get Track: `/tracks/{id}` [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-track) -Get Track's Audio Features - Deprecated: /audio-features/{id} [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-audio-features) +Get Track's Audio Features _(Deprecated)_: `/audio-features/{id}` [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-audio-features) -Get Track's Audio Analysis - Deprecated: /audio-analysis/{id} [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-audio-analysis) +Get Track's Audio Analysis _(Deprecated)_: `/audio-analysis/{id}` [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-audio-analysis) -Get Artist: /artists/{id} [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-an-artist) +Get Artist: `/artists/{id}` [Official Spotify Documentation](https://developer.spotify.com/documentation/web-api/reference/get-an-artist) + +## Docker usage + +`cd` inside the projects directory: +```sh +cd predictify +``` +To run predictify inside a container, first make sure to build the image: +```sh +make dockerfile +``` +> [!NOTE] +> +Create a seperate data directory (e.g. `docker-data`): +```sh +mkdir docker-data +``` +> [!NOTE] +> To detatch the container to run it in the background add the `--detach` directly after the `run` command. +Then run the following docker command, to run the container in the foreground: +```sh +docker run \ + --name predictify \ + --network=host \ + --volume $(pwd)/data-docker:/app/predictify/data \ + --volume $(pwd)/config:/app/predictify/config \ + predictify:unstable +``` ## Authors From aeb7c490687551347e743e3be64c30085537b5e9 Mon Sep 17 00:00:00 2001 From: Chris Kiriakou Date: Fri, 21 Mar 2025 19:07:30 +0100 Subject: [PATCH 5/8] Create improved Dockerfile with volume mounts for storing data * Add volume mounts to Dockerfile to enable persisten storing of database and tokens when using docker * Makefile helps with cleanup and building the Dockerfile * Startup fires the predictify runtime inside the container * `.dockerignore` contains files that are not needed by the docker, e.g. files that would otherwise slow build process down --- .dockerignore | 11 +++++++++++ Makefile | 19 +++++++++++++++++++ docker/Dockerfile | 35 +++++++++++++++++++++++++++++++++++ docker/startup.sh | 5 +++++ 4 files changed, 70 insertions(+) create mode 100644 .dockerignore create mode 100644 Makefile create mode 100644 docker/Dockerfile create mode 100755 docker/startup.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..99bd07d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +logs/ +data/ +src/__pycache__/ +.git +*.md +.venv +LICENSE +MAKEFILE +pytest.ini +test/ + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..89a5710 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +.PHONY: all dockerfile clean + +TAG="unstable" +PROJ_NAME="predictify" + +all: install dockerfile + +install: + mkdir -p ./data + +dockerfile: ./docker/Dockerfile + docker build \ + --tag "$(PROJ_NAME):$(TAG)" \ + --build-arg PROJ_NAME=$(PROJ_NAME) \ + --file ./docker/Dockerfile \ + . + +clean: ./spotify_scraped.db + rm -r ./data/spotify_scraped.db diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..5e86c2c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,35 @@ +FROM alpine:3.21.3 + +# Set environment variables +ARG PROJ_NAME +ENV PROJ_NAME=${PROJ_NAME} + +RUN mkdir -p /app/${PROJ_NAME} + +# The following steps are executed from the specified directory below +WORKDIR /app/${PROJ_NAME} + +# Install all necessary software +RUN apk add --no-cache python3 sqlite + +# Create the directories, needed for persistent storage (e.g. database, tokens) +RUN mkdir ./data ./src ./config + +# Create mount points for logs, data, src and config +VOLUME /var/log ./data ./src ./config + +# Copy the application source code +COPY ./src/ ./src/ + +# Create a seperate venv inside the container & install requirements +COPY ./requirements.txt ./requirements.txt +RUN \ + python -m venv .venv && \ + source .venv/bin/activate && \ + ./.venv/bin/pip install -r ./requirements.txt && \ + deactivate + +COPY ./docker/startup.sh ./startup.sh + +# When starting the contianer the following is executed +ENTRYPOINT ["./startup.sh"] diff --git a/docker/startup.sh b/docker/startup.sh new file mode 100755 index 0000000..f6092ad --- /dev/null +++ b/docker/startup.sh @@ -0,0 +1,5 @@ +#!/bin/sh +# +# Startup predictify. Don't use this. This is for docker specifically. +source .venv/bin/activate +.venv/bin/python src/runtime.py From 0531a346b5800c26ad8c327ba45e7d48313ff507 Mon Sep 17 00:00:00 2001 From: Chris Kiriakou Date: Fri, 21 Mar 2025 19:12:44 +0100 Subject: [PATCH 6/8] Add new folders --- .gitignore | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 6e93763..be40467 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,9 @@ # Test running file main_test.py -# databases -*.db - -# Custom Tokens file/rotator -tokens.json +# data dir +data/* +data-docker/ # Visual Studio Code .vscode/ From fd1d8e68742e3e0f9dd71a5a19179b2ebf3efbfc Mon Sep 17 00:00:00 2001 From: Chris Kiriakou Date: Fri, 21 Mar 2025 19:13:46 +0100 Subject: [PATCH 7/8] Change the directory location --- config/.env.example | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 config/.env.example diff --git a/config/.env.example b/config/.env.example new file mode 100644 index 0000000..545e283 --- /dev/null +++ b/config/.env.example @@ -0,0 +1,3 @@ +SPOTIFY_CLIENT_ID=your_token_here +SPOTIFY_CLIENT_SECRET=your_token_here +SPOTIFY_REDIRECT_URI=http://localhost:8888/callback From 71e9db8738bca9986ac4f3c97d093072c895ed10 Mon Sep 17 00:00:00 2001 From: Chris Kiriakou Date: Fri, 21 Mar 2025 19:17:12 +0100 Subject: [PATCH 8/8] Remove unused tag. --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 8f3b60b..b6a4366 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,6 @@ To run predictify inside a container, first make sure to build the image: ```sh make dockerfile ``` -> [!NOTE] -> Create a seperate data directory (e.g. `docker-data`): ```sh mkdir docker-data