valid matrix
Hey guys, have been trying to implement a bunch of proxies to my python script to effectively scrape web without facing issues pertained to rate-limiting etc.
I have bought a set of proxies, the web let's me download the updated version; I made a script to check if all of the ones are working before adding them to a list that my script (a Discord Bot) can use.
For my script I keep facing a ProxyError that I just can't get around, I have tried many ways to keep continuing with the code and wrote a couple Exceptions, however we will always get stuck and my script will stuck after fetching the Proxy that is also marked as down by the Host.
The logic of my script was just to send a basic request via that Proxy - if it returns a response within 3 seconds mark it as working, if not mark it as not working and remove from the file for further use.
Any idea?
import requests
from requests.exceptions import ProxyError, ConnectTimeout, ReadTimeout, RequestException
from urllib3.exceptions import MaxRetryError
read proxies from file
def proxyfile(file_path):
with open(file_path, 'r') as file:
proxies = [line.strip() for line in file.readlines() if line.strip()]
return proxies
testing
def test_proxy(proxy):
try:
ip, port, user, password = proxy.split(':')
proxy_url = f"http://{user}:{password}@{ip}:{port}"
proxies = {
'http': proxy_url,
'https': proxy_url
}
response = requests.get('http://httpbin.org/ip', proxies=proxies, timeout=5)
if response.status_code == 200:
print(f"Proxy {proxy} is working.")
else:
print(f"Proxy {proxy} is not working. Status Code: {response.status_code}")
except MaxRetryError:
print(f"Proxy {proxy} failed: Max retries exceeded.")
except ProxyError as e:
print(f"Proxy {proxy} failed due to proxy error: {e}")
except ConnectTimeout:
print(f"Proxy {proxy} failed due to connection timeout.")
except ReadTimeout:
print(f"Proxy {proxy} failed due to read timeout.")
except RequestException as e:
print(f"Proxy {proxy} failed: {e}")
#proxy file
proxies = proxyfile('cleaned_proxies.txt')
testing function
for proxy in proxies:
try:
test_proxy(proxy)
except Exception as e:
print(f"An unexpected error occurred with proxy {proxy}: {e}")
continue
twin stratus
