User-agent: *
Disallow: /search
Disallow: /email/
Disallow: /advisory/?*action=search*
Disallow: /blog/authors/
Disallow: /blog/tags/
Disallow: /media
Disallow: /radar/?id=
Disallow: /radar/api/
Disallow: /ssl/api/v1/get_subdomains/
Disallow: /ssl/api/v1/get_relevant_domains/
Disallow: /ssl/api/v1/get_third_party_result/
Disallow: /ssl/api/v1/get_certificate_expiration/
Disallow: /websec/api/v1/get_subdomains/
Disallow: /websec/api/v1/get_relevant_domains/
Disallow: /websec/api/v1/get_app_scan/
Disallow: /websec/api/v1/get_certificate_expiration/
Disallow: /websec/api/v1/get_third_party_result/
Disallow: /websec/api/v1/get_fingerprint/

User-agent: ia_archiver
Disallow: /

User-agent: Twitterbot
Allow: /ssl/?id=
Allow: /websec/?id=
Allow: /radar/?id=