bMorgan/sitemap-crawler
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

simplified config, added checksum-based lastmod check

Browse source
This commit is contained in:
bMorgan01 2022-09-13 14:47:10 -06:00
parent 64be0f1770
commit 9e688663e1
2 changed files with 69 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

20
crawl.conf Normal file → Executable file
View file

@ -1,9 +1,11 @@
# Domain Ex: benrmorgan.com
benrmorgan.com
# Prefix Ex: http://www.
http://www.
# Target path Ex /var/www/html/sitemap.xml or ./sitemaps/sitemap.xml
/var/www/html/sitemap.xml
# Ignore urls containing Ex: /files/
/files/
/images/
[Config]
; Domain Ex: benrmorgan.com
domain = benrmorgan.com
; Prefix Ex: http://www.
prefix = http://www.
; Target path Ex /var/www/html/sitemap.xml or ./sitemaps/sitemap.xml
target = /var/www/html/sitemap.xml
; Checksums path Ex ./checksums
checksums = ./checksums
; Ignore urls containing Ex: /files/
ignore = /files/, /images/