From 2339d78ad732be298f9bfffb6c16ce0883a3939f Mon Sep 17 00:00:00 2001 From: Askill Date: Sun, 11 Dec 2022 13:18:56 +0100 Subject: [PATCH] merged 2 if statements in crawler --- src/Crawler.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/Crawler.py b/src/Crawler.py index adf3f1e..e411a0d 100644 --- a/src/Crawler.py +++ b/src/Crawler.py @@ -71,13 +71,11 @@ class Crawler: n_links = [] for link in _links: - if link not in n_links: + if link not in n_links and level < limit: if link.startswith("http"): - if level < limit: - n_links.append((level+1, link)) + n_links.append((level+1, link)) else: - if level < limit: - n_links.append((level+1, urljoin(site.url, link))) + n_links.append((level+1, urljoin(site.url, link))) unchecked += n_links self._links[root] = n_links