how can i find all hidden tab hrefs during web scraping?

You can use this example to get links to documents from other tabs:

import requests
from bs4 import BeautifulSoup

url = ''
headers = {'X-MicrosoftAjax': 'Delta=true',
           'X-Requested-With': 'XMLHttpRequest'}

with requests.session() as s:

    soup = BeautifulSoup(s.get(url).content, 'html.parser')

    data = {}
    for i in'input[name]'):
        data[i['name']] = i.get('value', '')

    for page in range(0, 6):
        print('Tab no.{}..'.format(page))
        data['ScriptManager'] = "ScriptManager|dnn$ctr1843$View$rtSectionHearingTypes"
        data['__EVENTARGUMENT'] = '{"type":0,"index":"' + str(page) + '"}'
        data['__EVENTTARGET'] ="dnn$ctr1843$View$rtSectionHearingTypes"
        data['dnn_ctr1843_View_rtSectionHearingTypes_ClientState'] = '{"selectedIndexes":["' + str(page) + '"],"logEntries":[],"scrollState":{}}'
        data['__ASYNCPOST'] = "true"
        data['RadAJAXControlID'] = "dnn_ctr1843_View_RadAjaxManager1"

        soup = BeautifulSoup(, headers=headers, data=data).content, 'html.parser' )
        for a in'a[href*="documents"]'):
            print('' + a['href'])


Tab no.0..
Tab no.1..
Tab no.2..
Tab no.3..
Tab no.4..
Tab no.5..

CLICK HERE to find out more related problems solutions.

Leave a Comment

Your email address will not be published.

Scroll to Top