Merge pull request #4 from kristy-fournier/sqlite

Complete backend redesign to using SQLite

.gitignore

@@ -1,3 +1,3 @@
 server/sound/
-server/songDatabase.json
+*.db
 start.bat

Client/scripts.js

@@ -314,9 +314,9 @@ document.addEventListener('keydown', function(e){
 document.getElementById("playlist-mode").style.display = "none";
 document.getElementById("settings-mode").style.display = "none";
 //.ontouch for mobile??
-document.getElementById("volumerange").onchange = function() {
+document.getElementById("volumerange").onchange = async function() {
-    let returnValue = getFromServer({setting:"volume",level:this.value}, "settings")
+    let returnValue = await getFromServer({setting:"volume",level:this.value}, "settings")
-    if (returnValue !=0) {
+    if (returnValue["volumePassed"] !=0) {
         alertText("Nothing is playing")
         document.getElementById("volumerange").value = -1
     }

Server/databaseGenerator.py

@@ -1,7 +1,8 @@
 import os
 from mutagen.easyid3 import EasyID3
 from mutagen.mp3 import MP3
-import requests, ast, time, math, argparse, json
+import sqlite3 as sql
+import requests, ast, time, math, argparse
 
 loading = ["-","\\","|","/"]
 
@@ -22,32 +23,35 @@ else:
 # apikeylastfm = "KeyHere"
 # soundLocation = "directoryHere"
 songFiles = os.listdir(soundLocation)
+fileOfDB = sql.connect("songDatabase.db")
+songDatabase = fileOfDB.cursor()
+# setting song directory
+songDatabase.execute("CREATE TABLE IF NOT EXISTS meta (id TEXT PRIMARY KEY, data TEXT);")
+try:
+    songDatabase.execute("INSERT INTO meta (id, data) VALUES (?,?)",("songDirectory",soundLocation))
+except:
+    songDatabase.execute("UPDATE meta SET data = ? WHERE id = 'songDirectory'", (soundLocation,))
 if args.mode.lower() == "update":
-    try:   
+    #Create if not exists
-        with open('songDatabase.json', 'r') as handle:
+    songDatabase.execute("CREATE TABLE IF NOT EXISTS songs (filename TEXT PRIMARY KEY, title TEXT, artist TEXT, art TEXT, length INTEGER);")
-            songDatabaseList = json.load(handle)
+    songDatabase.execute("SELECT filename FROM songs;")
-    except:
+    dBfilelist = songDatabase.fetchall()
-        songDatabaseList={"songDirectory":soundLocation,'songData':{}}
+    dBfilelistSet = set()
-    deleteySongs = []
+    for i in dBfilelist:
-    for i in songDatabaseList["songData"]:
+        dBfilelistSet.add(i[0])
-        try:
+    # Delete nonexistant files
-            if songFiles.index(i) == -1:
+    deleteySongs = list(dBfilelistSet - set(songFiles))
-                deleteySongs.append(i)
+    songDatabase.executemany("DELETE FROM songs WHERE filename = ?", [(item,) for item in deleteySongs]) # in this line it turns the list of strings into a list of tuples of strings
-        except:
+    print("Deleted: " + ", ".join(deleteySongs)+ " from database")
-            deleteySongs.append(i)
+    # only include new files in list to be used
-    if deleteySongs:
+    songFiles = list(set(songFiles) - dBfilelistSet)
-        print("deleted: " + ", ".join(deleteySongs)+ " from database")
-    for i in deleteySongs:
-        songDatabaseList["songData"].pop(i)
-    for i in songDatabaseList["songData"]:
-        songFiles.remove(i)
-    # This prints everything in the directory, including non mp3s
-    # theres not agood way to fix this without looping again.
     print("new songs: " + ", ".join(songFiles))
 elif args.mode.lower()=="new":
-    songDatabaseList={"songDirectory":soundLocation,'songData':{}}
+    songDatabase.execute("DROP TABLE IF EXISTS songs;")
+    songDatabase.execute("CREATE TABLE songs (filename TEXT PRIMARY KEY, title TEXT, artist TEXT, art TEXT, length INTEGER);")
 else:
     raise ValueError("Must be \"new\" or \"update\"")
+
 if args.art.lower() == "true" and not(args.apikey == ""):
     x = len(songFiles)*0.25
     if x > 60:
@@ -65,12 +69,19 @@ for i in songFiles:
         title = song['title'][0]
         artist = song['artist'][0]
     except:
-        try:
+        if "_" in i:
             # if metadata is missing, try to use file name following title_artist.mp3
             song = i.split("_")
             title = song[0]
             artist = song[1].split(".")[0]
-        except:
+        elif "-" in i:
+            # if there's no underscore, try artist - title.mp3
+            song = i.split("-")
+            title = song[1].split(".")[0]
+            artist = song[0]
+            title = title.strip()
+            artist = artist.strip()
+        else:
             #if the file is not formatted with an underscore, the title is the file name
             title = i
             artist = None
@@ -95,7 +106,8 @@ for i in songFiles:
         index = (songFiles.index(i))%4
         print("\r" + str(loading[index] + str(math.floor((songFiles.index(i)/(len(songFiles)-1))*100))+ "%"), end='', flush=True)
     # each "song" is stored as a dictionary/JSON entry following the format seen in the readME
-    songDatabaseList["songData"][i] = ({"title":title,"artist":artist,"art":image,"length":length})
+    songDatabase.execute(f"INSERT INTO songs (filename, title, artist, art, length) VALUES (?,?,?,?,?)",(i,title,artist,image,length))
 
-with open('songDatabase.json', 'w') as handle:
+
-    json.dump(songDatabaseList, handle)
+
+fileOfDB.commit()

Server/webbyBits.py

@@ -1,7 +1,8 @@
 from flask import Flask
 from flask import request
 from flask_cors import CORS
-import json,vlc,threading,time,random, argparse
+import sqlite3 as sql
+import vlc,threading,time,random, argparse
 # Argparse Stuff
 parser=argparse.ArgumentParser(description="Options for the Webby Bits")
 # this is no longer needed assuming my file works correctly with the generator
@@ -9,10 +10,12 @@ parser=argparse.ArgumentParser(description="Options for the Webby Bits")
 parser.add_argument('-p','--port',help="Port to host on, not the same as the web (client) port",default='19054')
 portTheUserPicked=parser.parse_args().port
 
-# open the json file as a dictionary
+fileofDB = sql.connect("songDatabase.db")
-with open('./songDatabase.json', 'r') as handle:
+songDatabase = fileofDB.cursor()
-    songDatabaseList = json.load(handle)
+
-soundLocation = songDatabaseList["songDirectory"]
+#song directory
+songDatabase.execute("SELECT * FROM meta WHERE id='songDirectory';")
+soundLocation = songDatabase.fetchall()[0][1]
 
 if soundLocation[-1] == "/" or soundLocation[-1] == "\\":
     pass
@@ -20,6 +23,13 @@ elif "/" in soundLocation:
     soundLocation += "/"
 else:
     soundLocation += "\\"
+print(soundLocation)
+#Create Virtual table for searching
+songDatabase.execute("DROP TABLE virtualSongs;")
+songDatabase.execute("CREATE VIRTUAL TABLE virtualSongs USING fts5(filename, title, artist, art, length);")
+songDatabase.execute("INSERT INTO virtualSongs SELECT * FROM songs;")
+fileofDB.commit()
+fileofDB.close()
 #Initializing all the global stuff
 random.seed()
 global partyMode
@@ -55,7 +65,7 @@ def playQueuedSongs():
                 player.stop()
                 skipNow = False
                 songNext = playlist.pop(0)
-                media = fakeplayer.media_new("sound/"+songNext)
+                media = fakeplayer.media_new(soundLocation+songNext)
                 player.set_media(media)
                 player.play()
             elif (skipNow==True or (z == "State.Ended" or z == "State.NothingSpecial" or z=="State.Stopped")):
@@ -64,9 +74,13 @@ def playQueuedSongs():
                 songNext = None
                 player.stop()
             elif len(playlist)<1 and (partyMode == True):
+                fileofDB = sql.connect("songDatabase.db")
+                songDatabase = fileofDB.cursor()
+                songDatabase.execute("SELECT * FROM songs ORDER BY RANDOM() LIMIT 1;")
+                result = songDatabase.fetchall()
                 # adds the random songs for party mode
                 # the above 2 means this only applies if (a song is playing (or paused)) and the queue is empty
-                playlist.append(random.choice(songDatabaseList)["file"])
+                playlist.append(result[0][0])
         # check for new songs every second
         # I just didn't want to eat too much processing looping  
         time.sleep(1)
@@ -115,21 +129,27 @@ def settingsControl():
 @app.route("/search", methods=['POST'])
 def searchSongDB():
     recieveData=request.get_json(force=True)
-    tempData = {}
+    fileofDB = sql.connect("songDatabase.db")
+    songDatabase = fileofDB.cursor()
+    results = []
     if (recieveData['search'] == ""):
-        tempData = songDatabaseList["songData"].copy()
+        songDatabase.execute("SELECT * FROM virtualSongs")
+        results = songDatabase.fetchall()
     else:
-        for i in songDatabaseList["songData"]:
+        songDatabase.execute("SELECT * FROM virtualSongs WHERE virtualSongs MATCH ?",[recieveData['search']])
-            if ((songDatabaseList["songData"][i]["title"].lower().find(recieveData['search'].lower())) > -1):
+        results = songDatabase.fetchall()
-                tempData[i] = songDatabaseList["songData"][i]
+    tempdata = {}
-                
+    # this is a temporary solution so i dont have to change the 
-            try:
+    for i in results:
-                if (songDatabaseList["songData"][i]["artist"].lower().find(recieveData['search'].lower()) > -1):
+        tempdata[i[0]] = {
-                    tempData[i] = songDatabaseList["songData"][i]
+            "title": i[1],
-            except:
+            "artist": i[2],
-                pass
+            "art": i[3],
+            "length": i[4]
+        }
     # print(tempData)
-    return tempData
+    fileofDB.close()
+    return tempdata
 
 @app.route("/songadd", methods=["POST"])
 def songadd():
@@ -139,17 +159,36 @@ def songadd():
 @app.route("/playlist", methods=["POST"])
 def getPlaylist():
     global songNext
+    fileofDB = sql.connect("songDatabase.db")
+    songDatabase = fileofDB.cursor()
     tempPlaylist = []
     if songNext != None:
         # Adds the currently playing song
-        k = songDatabaseList["songData"][songNext]
+        songDatabase.execute("SELECT * FROM songs WHERE filename = ?",[songNext])
+        result = songDatabase.fetchall()[0]
+        # again, this is still using the old JSON format to avoid client changes
+        k = {
+            "title": result[1],
+            "artist": result[2],
+            "art": result[3],
+            "length": result[4]
+        }
         temp = k.copy()
         temp["playing"] = True
         temp["time"] = player.get_time()/1000
         tempPlaylist.append({songNext:temp})
     for i in playlist:
-        tempPlaylist.append({i:songDatabaseList["songData"][i]})
+        songDatabase.execute("SELECT * FROM songs WHERE filename = ?",[i])
+        result = songDatabase.fetchall()[0]
+        k = {
+            "title": result[1],
+            "artist": result[2],
+            "art": result[3],
+            "length": result[4]
+        }
+        tempPlaylist.append({i:k})
     # print(tempPlaylist)
+    fileofDB.close()
     return tempPlaylist
 
 if __name__ == "__main__":

readme.md

@@ -24,7 +24,7 @@ webbyBits.py
 1. Place mp3 files in the `sound/` folder
 2. Open `databaseGenerator.py` and put your LastFM API key in at the top or at runtime using `-k APIKey` (*optional*)
 3. Run `databaseGenerator.py`
-    * *The `databaseGenerator.py` will index all mp3 files, and save the information to `songDatabase.json`*
+    * *The `databaseGenerator.py` will index all mp3 files, and save the information to `songDatabase.db`*
     * *If getting images, this process may take a long time with a large amount of mp3 files*
 4. Run `webbyBits.py`
     * *The port can be customized at runtime using* `-p portNumber` *as an atribute*
@@ -39,7 +39,7 @@ These are specific details on each section of the app, and how to use them
 - `sound/` contains all mp3 files by default
 - `databaseGenerator.py` scans through mp3 files and gets information about them
     - `Filename, Title, Artist, Art, Length` are all saved 
-        - *If the title and artist are not in the mp3 metadata, it looks for a format of* `TITLE_ARTIST.mp3` *and otherwise defaults to the file name as the title, and no artist*
+        - *If the title and artist are not in the mp3 metadata, it looks for a format of* `TITLE_ARTIST.mp3` *then of* `ARTIST - TITLE.mp3` *and otherwise defaults to the file name as the title, and no artist*
         - Art is retrieved from LastFM
     - Running with `--mode (update/new)` either updates the current database and adds new songs/removes deleted songs, or recreates the entire database (update is default, and is faster in art mode)
     - Running with `--art (True/False)` retrieves art from  LastFM or doesn't (True is default)
@@ -50,19 +50,7 @@ These are specific details on each section of the app, and how to use them
         - Default `"./sound/"`
         - _This setting might be kinda iffy on Linux. You're on Linux just go and edit it if you have issues_
         - ~~__Make certain you only use forward slashes in your directory, even on Windows__~~ I think this should be fine now i'll check later
-- `songDatabase.json` stores all the information about each song in this format:
+- `songDatabase.db` stores all the information about each song in a SQLite database with tables `songs` and `meta`
-```
-{
-    "songDirectory": "./sound/", 
-    "songData": {
-        "Circus_Fox Szn.mp3": 
-        {"title": "Circus", 
-        "artist": "Fox Szn", 
-        "art": null, 
-        "length": 141}
-    }
-}
-```
 - `webbyBits.py` imports the database, runs all music playing, and accepts all commands from clients
     - Searches return matching songs
     - Accepts Play-Pause and Skip commands