User-agent: *
Disallow: /global_inc/
Allow: /global_inc/*.css
Allow: /global_inc/*.js

User-agent: *
Disallow: /global_engine/ajax/

# When crawlers hit the engine dir they sometimes publish confusing links to site content
# in their search results so we exclude these specific engines from crawling it.
# Note: Certain crawlers do need access to this directory so we do not want a blanket
# exlude statment here.
User-agent: *Google*
Allow: /global_engine/

User-agent: *googlebot*
Allow: /global_engine/

User-agent: *Slurp*
Allow: /global_engine/

User-agent: *sitecheck*
Allow: /global_engine/

User-agent: *mercator*
Allow: /global_engine/

User-agent: *ia_archiver*
Allow: /global_engine/

User-agent: *webcrawler*
Allow: /global_engine/

User-agent: *msnbot*
Allow: /global_engine/

User-agent: *scooter*
Allow: /global_engine/

Sitemap: https://www.masterbuilders.co.za/sitemap.xml