<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
    <url>
        <loc>https://example.com/page1</loc>
        <lastmod>2024-01-15</lastmod>
        <changefreq>weekly</changefreq>
        <priority>0.8</priority>
    </url>
    <url>
        <loc>https://example.com/page2</loc>
        <lastmod>2024-01-10</lastmod>
        <changefreq>daily</changefreq>
        <priority>1.0</priority>
    </url>
</urlset>

pip install beautifulsoup4 lxml requests

from bs4 import BeautifulSoup
import requests

def extract_urls_beautifulsoup(sitemap_url):
    response = requests.get(sitemap_url)
    soup = BeautifulSoup(response.content, 'xml')
    
    urls = []
    for loc in soup.find_all('loc'):
        urls.append(loc.text)
    
    return urls

sitemap_url = 'https://example.com/sitemap.xml'
all_urls = extract_urls_beautifulsoup(sitemap_url)

for url in all_urls:
    print(url)

from bs4 import BeautifulSoup
import requests

def parse_sitemap_complete(sitemap_url):
    response = requests.get(sitemap_url)
    soup = BeautifulSoup(response.content, 'xml')
    
    data = []
    for url_tag in soup.find_all('url'):
        url_dict = {}
        
        loc = url_tag.find('loc')
        if loc:
            url_dict['loc'] = loc.text
        
        lastmod = url_tag.find('lastmod')
        if lastmod:
            url_dict['lastmod'] = lastmod.text
        
        changefreq = url_tag.find('changefreq')
        if changefreq:
            url_dict['changefreq'] = changefreq.text
        
        priority = url_tag.find('priority')
        if priority:
            url_dict['priority'] = priority.text
        
        data.append(url_dict)
    
    return data

sitemap_url = 'https://example.com/sitemap.xml'
sitemap_data = parse_sitemap_complete(sitemap_url)

for item in sitemap_data:
    print(f"URL: {item.get('loc')}")
    print(f"Ngày cập nhật: {item.get('lastmod')}")
    print(f"Tần suất: {item.get('changefreq')}")
    print(f"Độ ưu tiên: {item.get('priority')}\n")

import xml.etree.ElementTree as ET

def parse_sitemap_elementtree(filepath):
    tree = ET.parse(filepath)
    root = tree.getroot()
    
    urls = []
    
    for url_element in root.findall('{http://www.sitemaps.org/schemas/sitemap/0.9}url'):
        loc_element = url_element.find('{http://www.sitemaps.org/schemas/sitemap/0.9}loc')
        if loc_element is not None:
            urls.append(loc_element.text)
    
    return urls

filepath = 'sitemap.xml'
all_urls = parse_sitemap_elementtree(filepath)

for url in all_urls:
    print(url)

import xml.etree.ElementTree as ET
import urllib.request

def parse_sitemap_index(sitemap_index_url):
    with urllib.request.urlopen(sitemap_index_url) as response:
        tree = ET.parse(response)
        root = tree.getroot()
    
    all_urls = []
    namespace = {'sitemap': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
    
    for sitemap in root.findall('sitemap:sitemap', namespace):
        loc = sitemap.find('sitemap:loc', namespace)
        if loc is not None:
            sitemap_url = loc.text
            print(f"Đang xử lý: {sitemap_url}")
            
            with urllib.request.urlopen(sitemap_url) as response:
                tree = ET.parse(response)
                sitemap_root = tree.getroot()
            
            for url_element in sitemap_root.findall('sitemap:url', namespace):
                url_loc = url_element.find('sitemap:loc', namespace)
                if url_loc is not None:
                    all_urls.append(url_loc.text)
    
    return all_urls

sitemap_index = 'https://example.com/sitemap_index.xml'
all_urls = parse_sitemap_index(sitemap_index)
print(f"Tổng số URL: {len(all_urls)}")

pip install lxml requests

from lxml import etree
import requests

def parse_sitemap_lxml(sitemap_url):
    response = requests.get(sitemap_url)
    root = etree.fromstring(response.content)
    
    namespace = {'sitemap': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
    urls = root.xpath('//sitemap:loc/text()', namespaces=namespace)
    
    return urls

sitemap_url = 'https://example.com/sitemap.xml'
urls = parse_sitemap_lxml(sitemap_url)

for url in urls:
    print(url)

from lxml import etree
import requests
import csv
from datetime import datetime

def extract_sitemap_lxml_advanced(sitemap_url):
    response = requests.get(sitemap_url)
    root = etree.fromstring(response.content)
    
    namespace = {'sitemap': 'http://www.sitemaps.org/schemas/sitemap/0.9'}
    
    data = []
    for url_element in root.xpath('//sitemap:url', namespaces=namespace):
        loc = url_element.xpath('sitemap:loc/text()', namespaces=namespace)[0]
        lastmod = url_element.xpath('sitemap:lastmod/text()', namespaces=namespace)
        changefreq = url_element.xpath('sitemap:changefreq/text()', namespaces=namespace)
        priority = url_element.xpath('sitemap:priority/text()', namespaces=namespace)
        
        data.append({
            'URL': loc,
            'Ngày cập nhật': lastmod[0] if lastmod else 'N/A',
            'Tần suất': changefreq[0] if changefreq else 'N/A',
            'Độ ưu tiên': priority[0] if priority else 'N/A'
        })
    
    return data

sitemap_url = 'https://example.com/sitemap.xml'
sitemap_data = extract_sitemap_lxml_advanced(sitemap_url)

with open('sitemap_data.csv', 'w', newline='', encoding='utf-8') as f:
    writer = csv.DictWriter(f, fieldnames=['URL', 'Ngày cập nhật', 'Tần suất', 'Độ ưu tiên'])
    writer.writeheader()
    writer.writerows(sitemap_data)

print(f"Đã lưu {len(sitemap_data)} URL vào file CSV")

pip install advertools pandas

import advertools as adv

sitemap_url = 'https://example.com/sitemap.xml'
sitemap_df = adv.sitemap_to_df(sitemap_url)

print(sitemap_df.head())
print(f"Tổng số URL: {len(sitemap_df)}")
print(sitemap_df.dtypes)

import advertools as adv
import pandas as pd
from datetime import datetime, timedelta

robots_url = 'https://example.com/robots.txt'
sitemap_df = adv.sitemap_to_df(robots_url)

print("Dữ liệu các cột:")
print(sitemap_df.dtypes)

if 'lastmod' in sitemap_df.columns:
    sitemap_df['lastmod'] = pd.to_datetime(sitemap_df['lastmod'], errors='coerce')
    
    thirty_days_ago = datetime.now() - timedelta(days=30)
    recent = sitemap_df[sitemap_df['lastmod'] >= thirty_days_ago]
    
    print(f"URL cập nhật trong 30 ngày qua: {len(recent)}")
    print("\nURL cập nhật gần đây:")
    print(recent[['loc', 'lastmod']].head(10))

if 'priority' in sitemap_df.columns:
    high_priority = sitemap_df[sitemap_df['priority'].astype(float) >= 0.8]
    print(f"\nURL có độ ưu tiên cao (>=0.8): {len(high_priority)}")

sitemap_df.to_csv('sitemap_analysis.csv', index=False)
print("\nDữ liệu đã được lưu vào file CSV")

import requests
from bs4 import BeautifulSoup

try:
    response = requests.get(
        'https://example.com/sitemap.xml',
        timeout=10
    )
    response.raise_for_status()
    soup = BeautifulSoup(response.content, 'xml')
except requests.exceptions.Timeout:
    print("Kết nối bị timeout!")
except requests.exceptions.RequestException as e:
    print(f"Lỗi: {e}")

import requests
import gzip
from io import BytesIO
from bs4 import BeautifulSoup

def parse_compressed_sitemap(url):
    response = requests.get(url)
    
    if url.endswith('.gz'):
        content = gzip.decompress(response.content)
    else:
        content = response.content
    
    soup = BeautifulSoup(content, 'xml')
    return soup

sitemap_url = 'https://example.com/sitemap.xml.gz'
soup = parse_compressed_sitemap(sitemap_url)

import requests
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor

def parse_individual_sitemap(url):
    try:
        response = requests.get(url, timeout=10)
        soup = BeautifulSoup(response.content, 'xml')
        urls = [loc.text for loc in soup.find_all('loc')]
        return urls
    except:
        return []

def parse_large_sitemap_index(index_url):
    response = requests.get(index_url)
    soup = BeautifulSoup(response.content, 'xml')
    
    sitemap_urls = [loc.text for loc in soup.find_all('loc')]
    
    all_urls = []
    with ThreadPoolExecutor(max_workers=5) as executor:
        results = executor.map(parse_individual_sitemap, sitemap_urls)
        for urls in results:
            all_urls.extend(urls)
    
    return all_urls

index_url = 'https://example.com/sitemap_index.xml'
all_urls = parse_large_sitemap_index(index_url)
print(f"Tổng URL: {len(all_urls)}")

import advertools as adv
import pandas as pd
import matplotlib.pyplot as plt

sitemap_url = 'https://example.com/sitemap.xml'
df = adv.sitemap_to_df(sitemap_url)

df['lastmod'] = pd.to_datetime(df['lastmod'], errors='coerce')
df['year_month'] = df['lastmod'].dt.to_period('M')

updates_per_month = df.groupby('year_month').size()

updates_per_month.plot(kind='bar', figsize=(12, 6))
plt.title('Số lượng cập nhật nội dung theo tháng')
plt.xlabel('Tháng')
plt.ylabel('Số lượng cập nhật')
plt.xticks(rotation=45)
plt.tight_layout()
plt.savefig('update_trends.png')
plt.show()

import advertools as adv
import pandas as pd
import requests
from concurrent.futures import ThreadPoolExecutor

def check_url_status(url):
    try:
        response = requests.head(url, timeout=5, allow_redirects=True)
        return url, response.status_code
    except:
        return url, 'Error'

sitemap_url = 'https://example.com/sitemap.xml'
df = adv.sitemap_to_df(sitemap_url)

urls = df['loc'].tolist()

status_codes = {}
with ThreadPoolExecutor(max_workers=10) as executor:
    results = executor.map(check_url_status, urls)
    for url, status in results:
        status_codes[url] = status

df['http_status'] = df['loc'].map(status_codes)

errors = df[df['http_status'] != 200]
print(f"URL có lỗi: {len(errors)}")
print(errors[['loc', 'http_status']].head(10))

import advertools as adv
import json
import csv

sitemap_url = 'https://example.com/sitemap.xml'
df = adv.sitemap_to_df(sitemap_url)

df.to_csv('sitemap_data.csv', index=False)

df.to_json('sitemap_data.json', orient='records', indent=2)

df.to_excel('sitemap_data.xlsx', index=False)

df_subset = df[['loc', 'priority']].to_dict('records')
with open('sitemap_simple.json', 'w') as f:
    json.dump(df_subset, f, indent=2, ensure_ascii=False)