Move hack/crawl under api/internal

api/internal/crawl/httpclient/httpclient.go

@@ -0,0 +1,23 @@
+package httpclient
+
+import (
+	"net/http"
+	"time"
+
+	"github.com/gomodule/redigo/redis"
+	"github.com/gregjones/httpcache"
+	rediscache "github.com/gregjones/httpcache/redis"
+)
+
+func FromCache(header http.Header) bool {
+	return header.Get(httpcache.XFromCache) != ""
+}
+
+func NewClient(conn redis.Conn) *http.Client {
+	etagCache := rediscache.NewWithClient(conn)
+	tr := httpcache.NewTransport(etagCache)
+	return &http.Client{
+		Transport: tr,
+		Timeout:   10 * time.Second,
+	}
+}