kar/siliconpin_spider
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

dbStructure

Browse Source
This commit is contained in:
Kar
2026-02-21 16:20:39 +05:30
parent 9056bbaf11
commit 54235ebff8
1 changed files with 13 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
main.go
View File

@@ -26,8 +26,10 @@ import (
// Constants & globals // Constants & globals
// ───────────────────────────────────────────────────────────────── // ─────────────────────────────────────────────────────────────────
const mainDBFile = "siliconpin_spider.sqlite" const appDataDir = "./data/app"
const skipDBFile = "skip_domain_list.sqlite" const domainDataDir = "./data/domain"
const mainDBFile = appDataDir + "/siliconpin_spider.sqlite"
const skipDBFile = appDataDir + "/skip_domain_list.sqlite"
var mainDB *sql.DB var mainDB *sql.DB
var skipDB *sql.DB var skipDB *sql.DB
@@ -295,7 +297,7 @@ func openDomainDB(domain string) (*sql.DB, error) {
return db, nil return db, nil
} }
db, err := sql.Open("sqlite3", domain+".sqlite?_journal=WAL&_busy_timeout=5000") db, err := sql.Open("sqlite3", domainDataDir+"/"+domain+".sqlite?_journal=WAL&_busy_timeout=5000")
if err != nil { if err != nil {
return nil, err return nil, err
} }
@@ -844,7 +846,7 @@ func addDomainHandler(w http.ResponseWriter, r *http.Request) {
"message": "domain added, crawler started", "message": "domain added, crawler started",
"domain": domain, "domain": domain,
"interval": interval, "interval": interval,
"db_file": domain + ".sqlite", "db_file": domainDataDir + "/" + domain + ".sqlite",
"sse": "/api/sse/" + domain, "sse": "/api/sse/" + domain,
}) })
} }
@@ -989,6 +991,12 @@ func broadcastAll(event string, data interface{}) {
func main() { func main() {
rand.Seed(time.Now().UnixNano()) //nolint:staticcheck rand.Seed(time.Now().UnixNano()) //nolint:staticcheck
if err := os.MkdirAll(appDataDir, 0o755); err != nil {
log.Fatalf("mkdir %s: %v", appDataDir, err)
}
if err := os.MkdirAll(domainDataDir, 0o755); err != nil {
log.Fatalf("mkdir %s: %v", domainDataDir, err)
}
if err := os.MkdirAll("static", 0o755); err != nil { if err := os.MkdirAll("static", 0o755); err != nil {
log.Fatalf("mkdir static: %v", err) log.Fatalf("mkdir static: %v", err)
} }
@@ -1060,7 +1068,7 @@ func main() {
for d, db := range domainDBs { for d, db := range domainDBs {
db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) //nolint:errcheck db.Exec(`PRAGMA wal_checkpoint(TRUNCATE)`) //nolint:errcheck
db.Close() db.Close()
log.Printf("checkpointed %s.sqlite", d) log.Printf("checkpointed %s/%s.sqlite", domainDataDir, d)
} }
domainDBsMu.RUnlock() domainDBsMu.RUnlock()