# Allow all crawlers access to certain pages.
User-agent: *
Allow: /

Host: https://matlabhelp.co.uk/
Sitemap: https://matlabhelp.co.uk/sitemap.xml

# Disallow Following crawlers access to site.

Disallow: /official/faqs/
Disallow: /official/our-process/
Disallow: /official/about-us/
Disallow: /contact.html 
Disallow: /event.html 
Disallow: /blog.html 
Disallow: /course.html 
Disallow: /teacher.html 
Disallow: /about.html 
Disallow: /index.html

User-agent: ia_archiver-web.archive.org
Disallow: /

User-agent: ia_archiver
Disallow: /

User-agent: TurnitinBot/2.1
Disallow: /

User-agent: TurnitinBot
Disallow: /

User-agent: SurveyBot
Disallow: /

User-agent: EmailWolf
Disallow: /

User-agent: EmailSiphon
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: AboutUsBot 
Disallow: /

User-agent: robtexbot
Disallow: /