727 lines
24 KiB
HTML
727 lines
24 KiB
HTML
<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Sitemap Generator</title>
|
|
<style>
|
|
* {
|
|
margin: 0;
|
|
padding: 0;
|
|
box-sizing: border-box;
|
|
}
|
|
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
min-height: 100vh;
|
|
padding: 20px;
|
|
}
|
|
|
|
.container {
|
|
max-width: 1200px;
|
|
margin: 0 auto;
|
|
background: white;
|
|
border-radius: 12px;
|
|
box-shadow: 0 20px 40px rgba(0,0,0,0.1);
|
|
overflow: hidden;
|
|
}
|
|
|
|
.header {
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
color: white;
|
|
padding: 30px;
|
|
text-align: center;
|
|
}
|
|
|
|
.header h1 {
|
|
font-size: 2.5rem;
|
|
margin-bottom: 10px;
|
|
}
|
|
|
|
.header p {
|
|
opacity: 0.9;
|
|
font-size: 1.1rem;
|
|
}
|
|
|
|
.main {
|
|
padding: 40px;
|
|
}
|
|
|
|
.form-section {
|
|
background: #f8f9fa;
|
|
padding: 30px;
|
|
border-radius: 8px;
|
|
margin-bottom: 30px;
|
|
}
|
|
|
|
.form-group {
|
|
margin-bottom: 20px;
|
|
}
|
|
|
|
label {
|
|
display: block;
|
|
margin-bottom: 8px;
|
|
font-weight: 600;
|
|
color: #333;
|
|
}
|
|
|
|
input[type="text"], input[type="number"] {
|
|
width: 100%;
|
|
padding: 12px;
|
|
border: 2px solid #e9ecef;
|
|
border-radius: 6px;
|
|
font-size: 1rem;
|
|
transition: border-color 0.3s;
|
|
}
|
|
|
|
input[type="text"]:focus, input[type="number"]:focus {
|
|
outline: none;
|
|
border-color: #667eea;
|
|
}
|
|
|
|
.btn {
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
color: white;
|
|
border: none;
|
|
padding: 12px 30px;
|
|
border-radius: 6px;
|
|
font-size: 1rem;
|
|
font-weight: 600;
|
|
cursor: pointer;
|
|
transition: transform 0.2s, box-shadow 0.2s;
|
|
}
|
|
|
|
.btn:hover {
|
|
transform: translateY(-2px);
|
|
box-shadow: 0 10px 20px rgba(102, 126, 234, 0.3);
|
|
}
|
|
|
|
.btn:disabled {
|
|
background: #6c757d;
|
|
cursor: not-allowed;
|
|
transform: none;
|
|
box-shadow: none;
|
|
}
|
|
|
|
.btn-secondary {
|
|
background: #28a745;
|
|
margin-left: 10px;
|
|
}
|
|
|
|
.btn-danger {
|
|
background: #dc3545;
|
|
margin-left: 10px;
|
|
}
|
|
|
|
.progress-section {
|
|
display: none;
|
|
background: #f8f9fa;
|
|
padding: 30px;
|
|
border-radius: 8px;
|
|
margin-bottom: 30px;
|
|
}
|
|
|
|
.progress-bar {
|
|
width: 100%;
|
|
height: 20px;
|
|
background: #e9ecef;
|
|
border-radius: 10px;
|
|
overflow: hidden;
|
|
margin-bottom: 15px;
|
|
}
|
|
|
|
.progress-fill {
|
|
height: 100%;
|
|
background: linear-gradient(90deg, #667eea, #764ba2);
|
|
width: 0%;
|
|
transition: width 0.3s ease;
|
|
border-radius: 10px;
|
|
}
|
|
|
|
.status {
|
|
padding: 15px;
|
|
border-radius: 6px;
|
|
margin-bottom: 20px;
|
|
font-weight: 500;
|
|
}
|
|
|
|
.status.info {
|
|
background: #d1ecf1;
|
|
color: rgb(12, 84, 96);
|
|
border: 1px solid #bee5eb;
|
|
}
|
|
|
|
.status.success {
|
|
background: #d4edda;
|
|
color: rgb(21, 87, 36);
|
|
border: 1px solid #c3e6cb;
|
|
}
|
|
|
|
.status.error {
|
|
background: #f8d7da;
|
|
color: rgb(114, 28, 36);
|
|
border: 1px solid #f5c6cb;
|
|
}
|
|
|
|
.log-section {
|
|
background: #2d3748;
|
|
color: #e2e8f0;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
font-family: 'Courier New', monospace;
|
|
font-size: 0.9rem;
|
|
max-height: 400px;
|
|
overflow-y: auto;
|
|
margin-bottom: 30px;
|
|
}
|
|
|
|
.log-entry {
|
|
margin-bottom: 10px;
|
|
padding: 8px;
|
|
border-radius: 4px;
|
|
}
|
|
|
|
.log-entry.start {
|
|
background: rgba(102, 126, 234, 0.2);
|
|
border-left: 3px solid #667eea;
|
|
}
|
|
|
|
.log-entry.progress {
|
|
background: rgba(40, 167, 69, 0.1);
|
|
border-left: 3px solid #28a745;
|
|
}
|
|
|
|
.log-entry.complete {
|
|
background: rgba(25, 135, 84, 0.1);
|
|
border-left: 3px solid #198754;
|
|
}
|
|
|
|
.log-entry.error {
|
|
background: rgba(220, 53, 69, 0.1);
|
|
border-left: 3px solid #dc3545;
|
|
}
|
|
|
|
.results-section {
|
|
display: none;
|
|
background: #f8f9fa;
|
|
padding: 30px;
|
|
border-radius: 8px;
|
|
}
|
|
|
|
.stats-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
gap: 20px;
|
|
margin-bottom: 30px;
|
|
}
|
|
|
|
.stat-card {
|
|
background: white;
|
|
padding: 20px;
|
|
border-radius: 8px;
|
|
box-shadow: 0 4px 6px rgba(0,0,0,0.1);
|
|
text-align: center;
|
|
}
|
|
|
|
.stat-number {
|
|
font-size: 2rem;
|
|
font-weight: bold;
|
|
color: #667eea;
|
|
}
|
|
|
|
.stat-label {
|
|
color: #6c757d;
|
|
margin-top: 5px;
|
|
}
|
|
|
|
.sites-list {
|
|
background: white;
|
|
border-radius: 8px;
|
|
overflow: hidden;
|
|
}
|
|
|
|
.site-item {
|
|
padding: 20px;
|
|
border-bottom: 1px solid #e9ecef;
|
|
display: flex;
|
|
justify-content: space-between;
|
|
align-items: center;
|
|
}
|
|
|
|
.site-item:last-child {
|
|
border-bottom: none;
|
|
}
|
|
|
|
.site-domain {
|
|
font-weight: 600;
|
|
color: #333;
|
|
margin-bottom: 5px;
|
|
}
|
|
|
|
.site-meta {
|
|
color: #6c757d;
|
|
font-size: 0.9rem;
|
|
}
|
|
|
|
.connection-status {
|
|
display: inline-block;
|
|
padding: 5px 10px;
|
|
border-radius: 4px;
|
|
font-size: 0.9rem;
|
|
margin-bottom: 15px;
|
|
}
|
|
|
|
.connection-status.connected {
|
|
background: #d4edda;
|
|
color: #155724;
|
|
}
|
|
|
|
.connection-status.disconnected {
|
|
background: #f8d7da;
|
|
color: #721c24;
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<div class="header">
|
|
<h1>🗺️ XML Sitemap Generator</h1>
|
|
<p>Generate sitemaps for your websites with real-time progress tracking</p>
|
|
</div>
|
|
|
|
<div class="main">
|
|
<!-- Input Form -->
|
|
<div class="form-section">
|
|
<div class="form-group">
|
|
<label for="siteUrl">Website URL</label>
|
|
<input type="text" id="siteUrl" placeholder="https://example.com" value="https://example.com">
|
|
</div>
|
|
|
|
<div class="form-group">
|
|
<label for="maxDepth">Max Crawl Depth (1-5)</label>
|
|
<input type="number" id="maxDepth" min="1" max="5" value="3">
|
|
</div>
|
|
|
|
<button class="btn" id="generateBtn" onclick="sitemapGen.generateSitemap()">
|
|
🚀 Generate Sitemap
|
|
</button>
|
|
</div>
|
|
|
|
<!-- Progress Section -->
|
|
<div class="progress-section" id="progressSection">
|
|
<div id="connectionStatus" class="connection-status disconnected">🔴 Disconnected</div>
|
|
|
|
<div class="progress-bar">
|
|
<div class="progress-fill" id="progressFill"></div>
|
|
</div>
|
|
|
|
<div class="status info" id="statusMessage">
|
|
Initializing...
|
|
</div>
|
|
|
|
<div class="stats-grid">
|
|
<div class="stat-card">
|
|
<div class="stat-number" id="totalPages">0</div>
|
|
<div class="stat-label">Pages Found</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number" id="currentDepth">0</div>
|
|
<div class="stat-label">Current Depth</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number" id="crawlTime">0s</div>
|
|
<div class="stat-label">Crawl Time</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div id="currentUrl" style="margin-bottom: 20px; font-size: 0.9rem; color: #6c757d;">
|
|
Current: -
|
|
</div>
|
|
|
|
<button class="btn btn-secondary" id="downloadBtn" onclick="sitemapGen.downloadSitemap()" style="display: none;">
|
|
📥 Download Sitemap
|
|
</button>
|
|
<button class="btn btn-danger" onclick="sitemapGen.clearAll()">
|
|
🗑️ Clear All
|
|
</button>
|
|
</div>
|
|
|
|
<!-- Log Section -->
|
|
<div class="log-section" id="logSection" style="display: none;">
|
|
<div id="logContainer"></div>
|
|
</div>
|
|
|
|
<!-- Results Section -->
|
|
<div class="results-section" id="resultsSection">
|
|
<h3 style="margin-bottom: 20px;">Previously Generated Sitemaps</h3>
|
|
<div class="sites-list" id="sitesList"></div>
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
class SitemapGenerator {
|
|
constructor() {
|
|
this.currentUUID = null;
|
|
this.currentSiteId = null;
|
|
this.eventSource = null;
|
|
this.startTime = null;
|
|
this.crawlTimer = null;
|
|
this.totalPages = 0;
|
|
|
|
// Load existing sites on init
|
|
this.loadExistingSites();
|
|
}
|
|
|
|
async generateSitemap() {
|
|
let url = document.getElementById('siteUrl').value.trim();
|
|
const maxDepth = parseInt(document.getElementById('maxDepth').value);
|
|
|
|
if (!url) {
|
|
alert('Please enter a website URL');
|
|
return;
|
|
}
|
|
|
|
// Auto-add protocol if missing
|
|
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
|
url = 'https://' + url;
|
|
document.getElementById('siteUrl').value = url;
|
|
}
|
|
|
|
try {
|
|
this.disableForm();
|
|
this.showProgress();
|
|
this.addLog(`Starting crawl of ${url}`, 'start');
|
|
|
|
const response = await fetch('/generate-sitemap-xml', {
|
|
method: 'POST',
|
|
headers: {
|
|
'Content-Type': 'application/json',
|
|
},
|
|
credentials: 'include',
|
|
body: JSON.stringify({
|
|
url: url,
|
|
max_depth: maxDepth
|
|
})
|
|
});
|
|
|
|
if (!response.ok) {
|
|
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
|
}
|
|
|
|
const result = await response.json();
|
|
this.currentUUID = result.uuid;
|
|
this.currentSiteId = result.site_id;
|
|
|
|
this.addLog(`UUID assigned: ${result.uuid}`, 'start');
|
|
this.addLog(`${result.message}`, 'start');
|
|
|
|
// Connect to stream with UUID
|
|
this.connectToStream(result.uuid);
|
|
|
|
} catch (error) {
|
|
this.addLog(`Failed to start crawl: ${error.message}`, 'error');
|
|
this.showError(error.message);
|
|
this.stopCrawlTimer();
|
|
this.enableForm();
|
|
}
|
|
}
|
|
|
|
connectToStream(uuid) {
|
|
if (this.eventSource) {
|
|
this.eventSource.close();
|
|
}
|
|
|
|
this.addLog(`Connecting to stream: /stream/${uuid}`, 'start');
|
|
|
|
try {
|
|
this.eventSource = new EventSource(`/stream/${uuid}`);
|
|
|
|
this.eventSource.addEventListener('connected', (e) => {
|
|
const data = JSON.parse(e.data);
|
|
this.addLog(`Connected to stream`, 'progress');
|
|
this.updateConnectionStatus(true);
|
|
document.getElementById('statusMessage').textContent = '🔄 Crawling...';
|
|
});
|
|
|
|
this.eventSource.addEventListener('started', (e) => {
|
|
const data = JSON.parse(e.data);
|
|
this.startTime = Date.now();
|
|
this.startCrawlTimer();
|
|
this.addLog(`Crawl started: ${data.url}`, 'start');
|
|
});
|
|
|
|
this.eventSource.addEventListener('progress', (e) => {
|
|
const data = JSON.parse(e.data);
|
|
this.updateProgress(data);
|
|
if (data.current_url) {
|
|
this.addLog(`Depth ${data.depth}: ${data.current_url}`, 'progress');
|
|
}
|
|
});
|
|
|
|
this.eventSource.addEventListener('complete', (e) => {
|
|
const data = JSON.parse(e.data);
|
|
this.completeGeneration(data);
|
|
this.addLog(`✅ Complete! Found ${data.pages_found} pages`, 'complete');
|
|
this.showDownloadButton();
|
|
this.eventSource.close();
|
|
this.stopCrawlTimer();
|
|
this.loadExistingSites();
|
|
});
|
|
|
|
this.eventSource.addEventListener('error', (e) => {
|
|
if (e.data) {
|
|
try {
|
|
const data = JSON.parse(e.data);
|
|
this.showError(data.error);
|
|
this.addLog(`❌ Error: ${data.error}`, 'error');
|
|
} catch (err) {
|
|
this.addLog('Stream error occurred', 'error');
|
|
}
|
|
}
|
|
this.eventSource.close();
|
|
this.updateConnectionStatus(false);
|
|
this.enableForm();
|
|
this.stopCrawlTimer();
|
|
});
|
|
|
|
this.eventSource.onerror = (e) => {
|
|
this.updateConnectionStatus(false);
|
|
this.addLog('Stream connection lost', 'error');
|
|
};
|
|
|
|
} catch (error) {
|
|
this.addLog(`Failed to connect to stream: ${error.message}`, 'error');
|
|
}
|
|
}
|
|
|
|
|
|
startCrawlTimer() {
|
|
this.stopCrawlTimer();
|
|
|
|
this.crawlTimer = setInterval(() => {
|
|
if (this.startTime) {
|
|
const elapsed = Math.floor((Date.now() - this.startTime) / 1000);
|
|
const element = document.getElementById('crawlTime');
|
|
if (element) {
|
|
element.textContent = `${elapsed}s`;
|
|
}
|
|
}
|
|
}, 1000);
|
|
}
|
|
|
|
stopCrawlTimer() {
|
|
if (this.crawlTimer) {
|
|
clearInterval(this.crawlTimer);
|
|
this.crawlTimer = null;
|
|
}
|
|
}
|
|
|
|
updateProgress(data) {
|
|
if (data.pages_found !== undefined) {
|
|
this.totalPages = data.pages_found;
|
|
document.getElementById('totalPages').textContent = data.pages_found;
|
|
}
|
|
|
|
if (data.depth !== undefined) {
|
|
document.getElementById('currentDepth').textContent = data.depth;
|
|
}
|
|
|
|
if (data.current_url) {
|
|
document.getElementById('currentUrl').textContent = `Current: ${data.current_url}`;
|
|
}
|
|
|
|
// Update progress bar (estimated)
|
|
const progress = Math.min((this.totalPages / 100) * 100, 90);
|
|
document.getElementById('progressFill').style.width = `${progress}%`;
|
|
}
|
|
|
|
completeGeneration(data) {
|
|
this.enableForm();
|
|
document.getElementById('progressFill').style.width = '100%';
|
|
document.getElementById('statusMessage').className = 'status success';
|
|
document.getElementById('statusMessage').textContent = `✅ Complete! Found ${data.pages_found || this.totalPages} pages`;
|
|
|
|
if (this.startTime) {
|
|
const elapsed = Math.floor((Date.now() - this.startTime) / 1000);
|
|
document.getElementById('crawlTime').textContent = `${elapsed}s`;
|
|
}
|
|
}
|
|
|
|
showDownloadButton() {
|
|
document.getElementById('downloadBtn').style.display = 'inline-block';
|
|
}
|
|
|
|
async downloadSitemap() {
|
|
if (!this.currentUUID) {
|
|
// Try to get the latest site
|
|
await this.loadExistingSites();
|
|
}
|
|
|
|
if (this.currentUUID) {
|
|
window.open(`/download/${this.currentUUID}`, '_blank');
|
|
} else {
|
|
alert('No sitemap available for download');
|
|
}
|
|
}
|
|
|
|
async loadExistingSites() {
|
|
try {
|
|
const response = await fetch('/sites');
|
|
if (!response.ok) return;
|
|
|
|
const sites = await response.json();
|
|
this.displaySites(sites);
|
|
|
|
if (sites.length > 0) {
|
|
this.currentUUID = sites[0].uuid;
|
|
this.currentSiteId = sites[0].id;
|
|
}
|
|
} catch (error) {
|
|
console.error('Failed to load sites:', error);
|
|
}
|
|
}
|
|
|
|
displaySites(sites) {
|
|
const container = document.getElementById('sitesList');
|
|
container.innerHTML = '';
|
|
|
|
if (sites.length === 0) {
|
|
container.innerHTML = '<p>No sitemaps generated yet.</p>';
|
|
document.getElementById('resultsSection').style.display = 'none';
|
|
return;
|
|
}
|
|
|
|
sites.forEach(site => {
|
|
const siteDiv = document.createElement('div');
|
|
siteDiv.className = 'site-item';
|
|
const createdDate = new Date(site.created_at).toLocaleString();
|
|
siteDiv.innerHTML = `
|
|
<div class="site-info">
|
|
<div class="site-domain">${site.domain}</div>
|
|
<div class="site-meta">
|
|
${site.page_count} pages •
|
|
Status: ${site.status} •
|
|
Created: ${createdDate}
|
|
</div>
|
|
</div>
|
|
<div>
|
|
<button class="btn btn-secondary" onclick="sitemapGen.downloadSiteSitemap('${site.uuid}')">
|
|
📥 Download
|
|
</button>
|
|
<button class="btn btn-danger" onclick="sitemapGen.deleteSite(${site.id})">
|
|
🗑️ Delete
|
|
</button>
|
|
</div>
|
|
`;
|
|
container.appendChild(siteDiv);
|
|
});
|
|
|
|
document.getElementById('resultsSection').style.display = 'block';
|
|
}
|
|
|
|
async downloadSiteSitemap(uuid) {
|
|
window.open(`/download/${uuid}`, '_blank');
|
|
}
|
|
|
|
async deleteSite(siteId) {
|
|
if (!confirm('Are you sure you want to delete this sitemap?')) return;
|
|
|
|
try {
|
|
const response = await fetch(`/sites/${siteId}`, {
|
|
method: 'DELETE'
|
|
});
|
|
|
|
if (response.ok) {
|
|
this.addLog(`Site ${siteId} deleted`, 'complete');
|
|
this.loadExistingSites();
|
|
} else {
|
|
throw new Error('Failed to delete site');
|
|
}
|
|
} catch (error) {
|
|
this.addLog(`Delete failed: ${error.message}`, 'error');
|
|
}
|
|
}
|
|
|
|
async clearAll() {
|
|
if (!confirm('Are you sure you want to clear all data? This cannot be undone.')) return;
|
|
|
|
try {
|
|
this.stopCrawlTimer();
|
|
|
|
const sitesResponse = await fetch('/sites');
|
|
if (sitesResponse.ok) {
|
|
const sites = await sitesResponse.json();
|
|
for (const site of sites) {
|
|
await fetch(`/sites/${site.id}`, {
|
|
method: 'DELETE'
|
|
});
|
|
}
|
|
}
|
|
|
|
this.addLog('All data cleared', 'complete');
|
|
this.loadExistingSites();
|
|
this.hideProgress();
|
|
} catch (error) {
|
|
this.addLog(`Clear failed: ${error.message}`, 'error');
|
|
}
|
|
}
|
|
|
|
showProgress() {
|
|
document.getElementById('progressSection').style.display = 'block';
|
|
document.getElementById('logSection').style.display = 'block';
|
|
}
|
|
|
|
hideProgress() {
|
|
document.getElementById('progressSection').style.display = 'none';
|
|
}
|
|
|
|
showError(message) {
|
|
document.getElementById('statusMessage').className = 'status error';
|
|
document.getElementById('statusMessage').textContent = `❌ Error: ${message}`;
|
|
}
|
|
|
|
addLog(message, type = 'info') {
|
|
const container = document.getElementById('logContainer');
|
|
const entry = document.createElement('div');
|
|
entry.className = `log-entry ${type}`;
|
|
const timestamp = new Date().toLocaleTimeString();
|
|
entry.innerHTML = `<strong>[${timestamp}]</strong> ${message}`;
|
|
|
|
container.prepend(entry);
|
|
container.scrollTop = 0;
|
|
}
|
|
|
|
updateConnectionStatus(connected) {
|
|
const status = document.getElementById('connectionStatus');
|
|
if (connected) {
|
|
status.className = 'connection-status connected';
|
|
status.textContent = '🟢 Connected';
|
|
} else {
|
|
status.className = 'connection-status disconnected';
|
|
status.textContent = '🔴 Disconnected';
|
|
}
|
|
}
|
|
|
|
disableForm() {
|
|
document.getElementById('generateBtn').disabled = true;
|
|
document.getElementById('generateBtn').textContent = '🔄 Generating...';
|
|
document.getElementById('siteUrl').disabled = true;
|
|
document.getElementById('maxDepth').disabled = true;
|
|
}
|
|
|
|
enableForm() {
|
|
document.getElementById('generateBtn').disabled = false;
|
|
document.getElementById('generateBtn').textContent = '🚀 Generate Sitemap';
|
|
document.getElementById('siteUrl').disabled = false;
|
|
document.getElementById('maxDepth').disabled = false;
|
|
}
|
|
}
|
|
|
|
// Initialize the application
|
|
const sitemapGen = new SitemapGenerator();
|
|
</script>
|
|
</body>
|
|
</html>
|