init

.gitignore

@@ -0,0 +1 @@
+update-unique-domains

go.mod

@@ -0,0 +1,8 @@
+module update-unique-domains
+
+go 1.21
+
+require (
+	github.com/go-sql-driver/mysql v1.7.1
+	golang.org/x/net v0.19.0
+)

go.sum

@@ -0,0 +1,4 @@
+github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
+github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=

main.go

@@ -0,0 +1,78 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"strings"
+
+	_ "github.com/go-sql-driver/mysql"
+	"golang.org/x/net/publicsuffix"
+)
+
+func main() {
+	// Connect to database
+	db, err := sql.Open("mysql", "sp:0000@tcp(l2:3306)/sp_spider")
+	if err != nil {
+		log.Fatalf("Failed to connect to database: %v", err)
+	}
+	defer db.Close()
+
+	// Test connection
+	if err := db.Ping(); err != nil {
+		log.Fatalf("Failed to ping database: %v", err)
+	}
+
+	// Query only unprocessed domains from unique_domains table
+	rows, err := db.Query("SELECT id, domain FROM unique_domains WHERE valid IS NULL")
+	if err != nil {
+		log.Fatalf("Failed to query domains: %v", err)
+	}
+	defer rows.Close()
+
+	// Process each domain one by one
+	for rows.Next() {
+		var id int
+		var domain string
+		if err := rows.Scan(&id, &domain); err != nil {
+			log.Printf("Failed to scan domain: %v", err)
+			continue
+		}
+
+		// Process domain
+		domainType := processDomain(domain)
+
+		// Update the domain record
+		updateQuery := "UPDATE unique_domains SET valid = ? WHERE id = ?"
+		_, err = db.Exec(updateQuery, domainType, id)
+		if err != nil {
+			log.Printf("Failed to update domain %s: %v", domain, err)
+		} else {
+			fmt.Printf("Updated ID %d: %s -> %s\n", id, domain, domainType)
+		}
+	}
+
+	if err = rows.Err(); err != nil {
+		log.Fatalf("Row iteration error: %v", err)
+	}
+}
+
+func processDomain(domain string) string {
+	domain = strings.TrimSpace(domain)
+	if domain == "" {
+		return "invalid"
+	}
+
+	// Get the effective TLD+1 (registrable domain)
+	etld1, err := publicsuffix.EffectiveTLDPlusOne(domain)
+	if err != nil {
+		return "invalid"
+	}
+
+	// Determine domain type
+	if domain == etld1 {
+		return "valid"
+	} else {
+		return "sub"
+	}
+}