Add a bigger delay between requests

src/crawler.ts

@@ -34,7 +34,7 @@ const spider = new Spider({
     // How many requests can be run in parallel
     concurrent: 10,
     // How long to wait after each request
-    delay: 100,
+    delay: 200,
     // A stream to where internal logs are sent, optional
     logs: logger,
     // Re-visit visited URLs, false by default
@@ -69,7 +69,10 @@ class Crawler extends Transform {
      */
     private timeout = 20;
 
-    private serverTimeout = 15 * 1000;
+    /**
+     * Timeout for a server
+     */
+    private serverTimeout = 30 * 1000;
     /**
      * Logger
      */