init
This commit is contained in:
253
db.go
Normal file
253
db.go
Normal file
@@ -0,0 +1,253 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sitemap-api/models"
|
||||
"time"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
type DB struct {
|
||||
conn *sql.DB
|
||||
}
|
||||
|
||||
func NewDB(dbPath string) (*DB, error) {
|
||||
conn, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
db := &DB{conn: conn}
|
||||
if err := db.createTables(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func (db *DB) Close() error {
|
||||
return db.conn.Close()
|
||||
}
|
||||
|
||||
func (db *DB) createTables() error {
|
||||
schema := `
|
||||
CREATE TABLE IF NOT EXISTS sites (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
uuid TEXT UNIQUE NOT NULL,
|
||||
domain TEXT NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
max_depth INTEGER DEFAULT 3,
|
||||
page_count INTEGER DEFAULT 0,
|
||||
status TEXT DEFAULT 'processing',
|
||||
|
||||
ip_address TEXT,
|
||||
user_agent TEXT,
|
||||
browser TEXT,
|
||||
browser_version TEXT,
|
||||
os TEXT,
|
||||
device_type TEXT,
|
||||
|
||||
session_id TEXT,
|
||||
cookies TEXT,
|
||||
referrer TEXT,
|
||||
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
completed_at DATETIME,
|
||||
last_crawled DATETIME
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS pages (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
site_id INTEGER NOT NULL,
|
||||
url TEXT NOT NULL UNIQUE,
|
||||
depth INTEGER DEFAULT 0,
|
||||
last_modified DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
priority REAL DEFAULT 0.5,
|
||||
change_freq TEXT DEFAULT 'monthly',
|
||||
FOREIGN KEY (site_id) REFERENCES sites(id) ON DELETE CASCADE
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
session_id TEXT UNIQUE NOT NULL,
|
||||
uuid TEXT,
|
||||
ip_address TEXT,
|
||||
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
last_activity DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
FOREIGN KEY (uuid) REFERENCES sites(uuid)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_uuid ON sites(uuid);
|
||||
CREATE INDEX IF NOT EXISTS idx_site_pages ON pages(site_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_session_id ON sessions(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_status ON sites(status);
|
||||
`
|
||||
|
||||
_, err := db.conn.Exec(schema)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) CreateSite(site *models.Site) (int, error) {
|
||||
query := `
|
||||
INSERT INTO sites (uuid, domain, url, max_depth, status, ip_address,
|
||||
user_agent, browser, browser_version, os, device_type, session_id,
|
||||
cookies, referrer, created_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
result, err := db.conn.Exec(query,
|
||||
site.UUID, site.Domain, site.URL, site.MaxDepth, site.Status,
|
||||
site.IPAddress, site.UserAgent, site.Browser, site.BrowserVersion,
|
||||
site.OS, site.DeviceType, site.SessionID, site.Cookies, site.Referrer,
|
||||
time.Now(),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
id, err := result.LastInsertId()
|
||||
return int(id), err
|
||||
}
|
||||
|
||||
func (db *DB) GetSiteByUUID(uuid string) (*models.Site, error) {
|
||||
query := `
|
||||
SELECT id, uuid, domain, url, max_depth, page_count, status,
|
||||
ip_address, user_agent, browser, browser_version, os, device_type,
|
||||
session_id, cookies, referrer, created_at, completed_at, last_crawled
|
||||
FROM sites WHERE uuid = ?
|
||||
`
|
||||
|
||||
site := &models.Site{}
|
||||
err := db.conn.QueryRow(query, uuid).Scan(
|
||||
&site.ID, &site.UUID, &site.Domain, &site.URL, &site.MaxDepth,
|
||||
&site.PageCount, &site.Status, &site.IPAddress, &site.UserAgent,
|
||||
&site.Browser, &site.BrowserVersion, &site.OS, &site.DeviceType,
|
||||
&site.SessionID, &site.Cookies, &site.Referrer, &site.CreatedAt,
|
||||
&site.CompletedAt, &site.LastCrawled,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("site not found")
|
||||
}
|
||||
|
||||
return site, err
|
||||
}
|
||||
|
||||
func (db *DB) GetSiteByID(id int) (*models.Site, error) {
|
||||
query := `
|
||||
SELECT id, uuid, domain, url, max_depth, page_count, status,
|
||||
ip_address, user_agent, browser, browser_version, os, device_type,
|
||||
session_id, cookies, referrer, created_at, completed_at, last_crawled
|
||||
FROM sites WHERE id = ?
|
||||
`
|
||||
|
||||
site := &models.Site{}
|
||||
err := db.conn.QueryRow(query, id).Scan(
|
||||
&site.ID, &site.UUID, &site.Domain, &site.URL, &site.MaxDepth,
|
||||
&site.PageCount, &site.Status, &site.IPAddress, &site.UserAgent,
|
||||
&site.Browser, &site.BrowserVersion, &site.OS, &site.DeviceType,
|
||||
&site.SessionID, &site.Cookies, &site.Referrer, &site.CreatedAt,
|
||||
&site.CompletedAt, &site.LastCrawled,
|
||||
)
|
||||
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, fmt.Errorf("site not found")
|
||||
}
|
||||
|
||||
return site, err
|
||||
}
|
||||
|
||||
func (db *DB) GetAllSites() ([]*models.Site, error) {
|
||||
query := `
|
||||
SELECT id, uuid, domain, url, max_depth, page_count, status,
|
||||
ip_address, user_agent, browser, browser_version, os, device_type,
|
||||
session_id, cookies, referrer, created_at, completed_at, last_crawled
|
||||
FROM sites ORDER BY created_at DESC
|
||||
`
|
||||
|
||||
rows, err := db.conn.Query(query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
sites := []*models.Site{}
|
||||
for rows.Next() {
|
||||
site := &models.Site{}
|
||||
err := rows.Scan(
|
||||
&site.ID, &site.UUID, &site.Domain, &site.URL, &site.MaxDepth,
|
||||
&site.PageCount, &site.Status, &site.IPAddress, &site.UserAgent,
|
||||
&site.Browser, &site.BrowserVersion, &site.OS, &site.DeviceType,
|
||||
&site.SessionID, &site.Cookies, &site.Referrer, &site.CreatedAt,
|
||||
&site.CompletedAt, &site.LastCrawled,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sites = append(sites, site)
|
||||
}
|
||||
|
||||
return sites, nil
|
||||
}
|
||||
|
||||
func (db *DB) UpdateSiteStatus(uuid string, status string, pageCount int) error {
|
||||
query := `
|
||||
UPDATE sites
|
||||
SET status = ?, page_count = ?, completed_at = ?, last_crawled = ?
|
||||
WHERE uuid = ?
|
||||
`
|
||||
|
||||
now := time.Now()
|
||||
_, err := db.conn.Exec(query, status, pageCount, now, now, uuid)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) DeleteSite(id int) error {
|
||||
// Pages will be deleted automatically due to CASCADE
|
||||
_, err := db.conn.Exec("DELETE FROM sites WHERE id = ?", id)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) AddPage(page *models.Page) error {
|
||||
query := `
|
||||
INSERT OR IGNORE INTO pages (site_id, url, depth, last_modified, priority, change_freq)
|
||||
VALUES (?, ?, ?, ?, ?, ?)
|
||||
`
|
||||
|
||||
_, err := db.conn.Exec(query,
|
||||
page.SiteID, page.URL, page.Depth, page.LastModified,
|
||||
page.Priority, page.ChangeFreq,
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (db *DB) GetPagesBySiteID(siteID int) ([]*models.Page, error) {
|
||||
query := `
|
||||
SELECT id, site_id, url, depth, last_modified, priority, change_freq
|
||||
FROM pages WHERE site_id = ? ORDER BY depth, url
|
||||
`
|
||||
|
||||
rows, err := db.conn.Query(query, siteID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
pages := []*models.Page{}
|
||||
for rows.Next() {
|
||||
page := &models.Page{}
|
||||
err := rows.Scan(
|
||||
&page.ID, &page.SiteID, &page.URL, &page.Depth,
|
||||
&page.LastModified, &page.Priority, &page.ChangeFreq,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pages = append(pages, page)
|
||||
}
|
||||
|
||||
return pages, nil
|
||||
}
|
||||
Reference in New Issue
Block a user