More database stuff

master
Noah Pederson 6 years ago
parent 4d985c5c7f
commit 95b8557a7a
  1. 5
      .idea/libraries/GOPATH__gocomics_.xml
  2. 103
      database/database.go
  3. 11
      gocomics.go
  4. 6
      gocomics.iml
  5. 12
      models/comics.go
  6. 55
      scanner/comicscanner.go
  7. 139
      utils/utils.go
  8. 7
      web/api.go
  9. 13
      web/comicstreamer.go

@ -1,10 +1,11 @@
<component name="libraryTable">
<library name="GOPATH &lt;gocomics&gt;">
<CLASSES>
<root url="file://$PROJECT_DIR$/../../../gopkg.in" />
<root url="file://$PROJECT_DIR$/../../../github.com" />
<root url="file://$PROJECT_DIR$/../../../golang.org" />
<root url="file://$PROJECT_DIR$/../.." />
<root url="file://$PROJECT_DIR$/../../../golang.org" />
<root url="file://$PROJECT_DIR$/../../../9fans.net" />
<root url="file://$PROJECT_DIR$/../../../gopkg.in" />
</CLASSES>
<JAVADOC />
<SOURCES />

@ -2,17 +2,20 @@ package database
import (
"database/sql"
"git.chiefnoah.tech/chiefnoah/gocomics/models"
_ "github.com/mattn/go-sqlite3"
"fmt"
"log"
"os"
"strings"
"git.chiefnoah.tech/chiefnoah/gocomics/models"
_ "github.com/mattn/go-sqlite3"
)
//holds a reference to a database connection and a transaction used for large database processes
//like adding a shitton of comics ;P
type Dbhandler struct {
Transaction *sql.Tx
Db *sql.DB
Db *sql.DB
}
func Init() {
@ -49,9 +52,17 @@ func Init() {
'ID' INTEGER PRIMARY KEY AUTOINCREMENT,
'Author' TEXT,
'Publisher' TEXT,
'Other' TEXT,
'Other' TEXT,
'ComicID' INTEGER,
FOREIGN KEY('ComicID') REFERENCES Comic(ID)
);`
var CREATE_CATEGORY string = `CREATE TABLE IF NOT EXISTS 'Category' (
'ID' INTEGER PRIMARY KEY AUTOINCREMENT,
'Name' TEXT NOT NULL UNIQUE,
'Parent' INTEGER NOT NULL,
'IsRoot' INTEGER DEFAULT 0,
'Full' TEXT NOT NULL UNIQUE,
FOREIGN KEY('Parent') REFERENCES Category(ID)
);`
var CREATE_COMIC_FILE string = `CREATE TABLE IF NOT EXISTS 'ComicFile' (
'ID' INTEGER PRIMARY KEY AUTOINCREMENT,
@ -68,13 +79,15 @@ func Init() {
'IssueNumber' REAL DEFAULT 0.0,
'PageCount' INTEGER,
'ComicFileID' INTEGER,
'Hash' TEXT NOT NULL,
'Hash' TEXT NOT NULL UNIQUE,
'Volume' TEXT,
'DateAdded' INTEGER DEFAULT 0,
'PublishDate' INTEGER,
'Synopsis' TEXT,
'Rating' REAL DEFAULT 0.0,
'Status' TEXT,
'CategoryID' INTEGER,
FOREIGN KEY('CategoryID') REFERENCES 'Category'('ID')
FOREIGN KEY('ComicFileID') REFERENCES 'ComicFile'('ID')
);`
var CREATE_CHARACTERS_BRIDGE string = `CREATE TABLE IF NOT EXISTS 'CharactersBridge' (
@ -122,6 +135,10 @@ func Init() {
if err != nil {
log.Fatal(err)
}
_, err = db.Exec(CREATE_CATEGORY)
if err != nil {
log.Fatal(err)
}
_, err = db.Exec(CREATE_COMIC_FILE)
if err != nil {
log.Fatal(err)
@ -151,14 +168,41 @@ func Init() {
}
func(h *Dbhandler) AddComic(comic models.ComicInfo, file models.ComicFile) bool {
func (h *Dbhandler) AddCategory(category *models.Category) error {
//I'm guessing the "OR REPLACE" part will cause issues later one when people don't want changing the directory
//structure of their comics to lose their categories. It might not be a problem if categories are purely based on
//the actual directory structure, but if the are arbitrarily set (which I plan on letting you do) it will have to
//rely on the CategoryID field of the Comic table, which won't get updated when a row is replaced because it generates
//a new ID. A possible solution is to use SQL triggers, but I'm unfamiliar with them, I'll have to look into it.
INSERT_CATEGORY := `INSERT OR REPLACE INTO Category(Name, Parent, IsRoot, Full) VALUES(?, (SELECT ID FROM Category WHERE Name = ?), ?, (SELECT Full FROM Category WHERE Name = ?) || ? || ?)`
stmt, err := h.Transaction.Prepare(INSERT_CATEGORY)
if err != nil {
log.Println("Error preparing statement: ", err)
return err
}
defer stmt.Close()
_, err = stmt.Exec(category.Name, category.Parent, category.IsRoot, category.Parent, fmt.Sprintf("%c", os.PathSeparator), category.Name)
if err != nil {
log.Println("Error adding category: ", err)
}
return err
}
func (h *Dbhandler) AddComic(comic *models.ComicInfo, file *models.ComicFile) error {
//The Hash field is shared because I felt like it. It doesn't really need to be shared but it helps to have a
//uniquely identifying field besides the ID generated by the database
INSERT_COMIC_FILE_INFO := `INSERT INTO ComicFile(RelativePath, AbsolutePath, FileName, Hash, Filesize) VALUES(?, ?, ?, ?, ?)`
INSERT_COMIC_INFO := `INSERT INTO Comic(Title, Series, IssueNumber, PageCount, ComicFileID, Hash, Volume, DateAdded, PublishDate, Synopsis, Rating, Status) VALUES (?, ?, ?, ?, (SELECT ID FROM ComicFile WHERE ComicFile.Hash = ?), ?, ?, ?, ?, ?, ?, ?)`
stmt, err := h.Transaction.Prepare(INSERT_COMIC_FILE_INFO)
defer stmt.Close()
if err != nil {
log.Fatal(err)
log.Println(err)
return err
}
_, err = stmt.Exec(file.RelativePath, file.AbsolutePath, file.FileName, file.Hash, file.FileSize)
if err != nil {
@ -169,22 +213,53 @@ func(h *Dbhandler) AddComic(comic models.ComicInfo, file models.ComicFile) bool
st, err := h.Transaction.Prepare(sql)
defer st.Close()
if err != nil {
log.Fatal("Unable to prepare statement: ", err)
log.Println("Unable to prepare statement: ", err)
return err
}
log.Printf("Updating with:%+v\n", file)
//log.Printf("Updating with:%+v\n", file)
_, err = st.Exec(file.RelativePath, file.AbsolutePath, file.FileName, file.Hash)
if err != nil {
log.Fatal("error updating: ", err)
log.Println("error updating: ", err)
return err
}
} else {
log.Fatal("Error inserting: ", err)
log.Println("Error inserting: ", err)
return err
}
}
stmt2, err := h.Transaction.Prepare(INSERT_COMIC_INFO)
defer stmt2.Close()
if err != nil {
log.Println(err)
return err
}
_, err = stmt2.Exec(comic.Title, comic.Series, comic.IssueNumber, comic.PageCount, file.Hash, file.Hash, comic.Volume,
comic.DateAdded, comic.PublishDate, comic.Synopsis, comic.Rating, comic.Status)
if err != nil {
if strings.Contains(err.Error(), "UNIQUE") {
//this means the record already exists. We don't want to overwrite tags here so we do nothing
log.Println("Already have metadata on: ", comic.Title, " ", comic.Series)
} else {
log.Println("Database error: ", err)
return err
}
}
//log.Printf("Results:%+v\n", res)
return false
return nil
}
func (h *Dbhandler) ExecuteSql(sql string, params ...interface{}) error {
stmt, err := h.Transaction.Prepare(sql)
if err != nil {
log.Println("Unable to begin statement in transaction: ", err)
return err
}
_, err = stmt.Exec(params...)
return err
}
//Creates a new dbhandler object for running a transaction
@ -202,8 +277,8 @@ func BeginTransaction() *Dbhandler {
return &handler
}
func(h *Dbhandler) FinishTransaction() error {
func (h *Dbhandler) FinishTransaction() error {
err := h.Transaction.Commit()
defer h.Db.Close()
return err
}
}

@ -2,8 +2,8 @@ package main
import (
"git.chiefnoah.tech/chiefnoah/gocomics/config"
"git.chiefnoah.tech/chiefnoah/gocomics/web"
"git.chiefnoah.tech/chiefnoah/gocomics/scanner"
"git.chiefnoah.tech/chiefnoah/gocomics/web"
"git.chiefnoah.tech/chiefnoah/gocomics/database"
"log"
@ -12,18 +12,17 @@ import (
//Let's get started!
func main() {
f, err := os.OpenFile("log.txt", os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
f, err := os.OpenFile("log.txt", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
if err != nil {
log.Fatal("Can't write log file! D:")
}
defer f.Close()
log.SetOutput(f)
//log.SetOutput(f)
database.Init()
config := &config.ApiConfig{
false, false, ":3008", ":3000",
}
go comicscanner.Scan(`F:\eBooks\eComics\Manga`)
UseTLS: true, ForceTLS: false, SSLPort: ":3008", HttpPort: ":3000"}
go comicscanner.Scan(`./comics`)
web.Start(config)
}

@ -2,7 +2,11 @@
<module type="GO_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="true">
<exclude-output />
<content url="file://$MODULE_DIR$" />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/.images" />
<excludeFolder url="file://$MODULE_DIR$/.temp" />
<excludeFolder url="file://$MODULE_DIR$/comics" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="GOPATH &lt;gocomics&gt;" level="project" />

@ -56,6 +56,18 @@ type UserProgress struct {
LastReadPage int `json:"last_read_page"`
}
//Used to create a pseudo directory structure.
//All comics must belong to a category. The default behavior
//is to make each folder that is walked a category
type Category struct {
ID int `json:"id"`
Name string `json:"name"`
Parent string `json:"parent"`
ParentId int `json:"parent_id"`
IsRoot bool `json:"is_root"`
Full string `json:"full"`
}
/*
ComicStreamer compatibility structures
*/

@ -3,16 +3,17 @@ package comicscanner
import (
//"github.com/fsnotify/fsnotify"
"crypto/md5"
"encoding/hex"
"fmt"
"git.chiefnoah.tech/chiefnoah/gocomics/database"
"git.chiefnoah.tech/chiefnoah/gocomics/models"
"git.chiefnoah.tech/chiefnoah/gocomics/utils"
"io/ioutil"
"log"
"os"
"path"
"path/filepath"
"strings"
"log"
"git.chiefnoah.tech/chiefnoah/gocomics/models"
"git.chiefnoah.tech/chiefnoah/gocomics/database"
"encoding/hex"
)
var root string
@ -20,9 +21,17 @@ var dbhandler *database.Dbhandler
func Scan(f string) error {
root = f
base := filepath.Base(f)
//models.Category{ID: 1, Name: base, Parent: 1, IsRoot:true, Full: base}
//generates the temp and image directories
setupDirs()
dbhandler = database.BeginTransaction()
err := dbhandler.ExecuteSql(`INSERT OR IGNORE INTO Category(ID, Name, Parent, IsRoot, Full) VALUES(?, ?, ?, ?, ?)`, 1, base, 1, true, base)
if err != nil {
log.Println("Error creating start category dir: ", err)
}
defer dbhandler.FinishTransaction()
err := filepath.Walk(f, visit)
err = filepath.Walk(f, visit)
if err != nil {
fmt.Printf("walk error: %v\n", err)
return err
@ -46,25 +55,36 @@ func visit(p string, f os.FileInfo, e error) error {
var comicfile models.ComicFile
//TODO: somehow get comic info based on filename/directory structure
//TODO: generate cover images using hash
checksum := md5.Sum(file)
n := len(checksum)
comicfile.Hash = hex.EncodeToString(checksum[:n])
comicfile.FileSize = int64(f.Size())
rel, _ := filepath.Rel(root, p)
comicfile.RelativePath = filepath.ToSlash(rel)
comicfile.RelativePath = filepath.Dir(filepath.ToSlash(filepath.Join(root, rel)))
comicfile.FileName = f.Name()
if !path.IsAbs(root) {
ab, err := filepath.Abs(p)
if err != nil {
log.Print("Couldn't get relative path: ", err)
}
comicfile.AbsolutePath = filepath.ToSlash(ab)
comicfile.AbsolutePath = filepath.Dir(filepath.ToSlash(ab))
}
//fmt.Printf("MD5: %s\n", comicfile.Hash)
dbhandler.AddComic(models.ComicInfo{}, comicfile)
dbhandler.AddComic(&models.ComicInfo{}, &comicfile)
go generateCoverImage(&comicfile)
} else {
//TODO: check if directory, if directory and directory=category is enabled (default), add it as a category
//limitation: category names (ie. directory names) MUST be unique.
if p == root {
//If the current directory is the root directory, we don't want to do anything
return nil
}
dir := filepath.Base(filepath.Dir(p))
name := filepath.Base(p)
category := models.Category{Name: name, Parent: dir, IsRoot: false}
dbhandler.AddCategory(&category)
}
return nil
}
@ -72,3 +92,20 @@ func visit(p string, f os.FileInfo, e error) error {
func watch(f []string) error {
return nil
}
//TODO: read metadata from file or filename
func setupDirs() {
os.MkdirAll(utils.IMAGES_DIR, 0755)
os.MkdirAll(utils.CACHE_DIR, 0755)
}
func generateCoverImage(comicfile *models.ComicFile) {
if _, f := os.Stat(filepath.Join(utils.IMAGES_DIR, comicfile.Hash)); os.IsNotExist(f) {
fmt.Println("No cover image found, generating")
err := utils.ExtractCoverImage(comicfile)
if err != nil {
log.Println("Extraction error: ", err)
}
}
}

@ -0,0 +1,139 @@
package utils
import (
"archive/zip"
"fmt"
"git.chiefnoah.tech/chiefnoah/gocomics/models"
"io"
"log"
"os"
"path/filepath"
)
const (
IMAGES_DIR = ".images"
CACHE_DIR = ".temp"
)
//based off answer here https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
//Extracts a cbz to the .temp directory inside a folder with the same name
func ExtractComic(comicfile *models.ComicFile) error {
r, err := zip.OpenReader(comicfile.AbsolutePath)
if err != nil {
log.Print("Unable to extract cbz\n")
return err
}
defer func() {
if err := r.Close(); err != nil {
log.Fatal(err)
}
}()
extractAndWrite := func(f *zip.File) error {
rc, err := f.Open()
if err != nil {
return err
}
defer func() {
if err := rc.Close(); err != nil {
log.Fatal(err)
}
}()
wd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
dirname := filepath.Join(wd, CACHE_DIR, comicfile.Hash)
os.MkdirAll(dirname, 0755)
path := filepath.Join(dirname, f.Name)
fmt.Println("Extracting to: ", path)
//This probably isn't necessary because we're always dealing with .cbz/.zip files
if f.FileInfo().IsDir() {
os.MkdirAll(path, f.Mode())
} else {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}()
_, err = io.Copy(f, rc)
if err != nil {
log.Fatal(err)
}
}
return nil
}
for _, f := range r.File {
//we only want the first file, so only extract that one
err = extractAndWrite(f)
if err != nil {
return err
}
}
return nil
}
func ExtractCoverImage(comicfile *models.ComicFile) error {
r, err := zip.OpenReader(comicfile.AbsolutePath)
if err != nil {
log.Print("Unable to extract cbz\n")
return err
}
defer func() {
if err := r.Close(); err != nil {
log.Fatal(err)
}
}()
extractAndWrite := func(f *zip.File) error {
rc, err := f.Open()
if err != nil {
return err
}
defer func() {
if err := rc.Close(); err != nil {
log.Fatal(err)
}
}()
wd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
dirname := filepath.Join(wd, IMAGES_DIR)
path := filepath.Join(dirname, comicfile.Hash)
fmt.Println("Extracting to: ", path)
//This probably isn't necessary because we're always dealing with .cbz/.zip files
if f.FileInfo().IsDir() {
os.MkdirAll(path, f.Mode())
} else {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return err
}
defer func() {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}()
_, err = io.Copy(f, rc)
if err != nil {
log.Fatal(err)
}
}
return nil
}
err = extractAndWrite(r.File[0])
if err != nil {
return err
}
return nil
}

@ -16,9 +16,14 @@ func Start(c *config.ApiConfig) {
router.GET("/", rootHandler)
router.GET("/dbinfo", dbInfoHandler)
router.GET("/version", versionHandler)
router.RunTLS(c.HttpPort, "./test.pem", "./test.key")
router.GET("/folders/*path", foldersHandler)
router.Run(c.HttpPort)
router.RunTLS(c.SSLPort, "./test.pem", "./test.key")
}
func rootHandler(c *gin.Context) {
c.String(http.StatusOK, "hi")
}

@ -5,6 +5,13 @@ import (
"net/http"
)
/*
Comic Streamer compatibility API endpoints and stuff goes here
*/
func dbInfoHandler(c *gin.Context) {
c.String(http.StatusOK, `{"comic_count": 13398, "last_updated": "2015-08-31T20:16:58.035000", "id": "f03b53dbd5364377867227e23112d3c7", "created": "2015-06-18T19:13:35.030000"}`)
}
@ -16,3 +23,9 @@ func versionHandler(c *gin.Context) {
func comicListHandler(c *gin.Context) {
}
func foldersHandler(c *gin.Context) {
path := c.Param("path")
c.String(http.StatusOK, "Test: " + path)
}
Loading…
Cancel
Save