add replacement filter to support replacmenttransformer

This commit is contained in:
Natasha Sarkar
2021-03-17 13:51:31 -07:00
parent 710db98dbf
commit fa0b237178
14 changed files with 722 additions and 103 deletions

View File

@@ -81,7 +81,7 @@ func (gc githubCrawler) Crawl(ctx context.Context,
output chan<- crawler.CrawledDocument, seen utils.SeenMap) error {
ranges := []RangeWithin{
RangeWithin{
{
start: uint64(0),
end: githubMaxFileSize,
},

View File

@@ -96,6 +96,6 @@ func TestRangeSizes(t *testing.T) {
returnedResult := RangeSizes(s)
expectedResult := RangeWithin{uint64(2365), uint64(10000)}
if !reflect.DeepEqual(returnedResult, expectedResult) {
t.Errorf("RangeSizes expected (%v), got (%v)",expectedResult, returnedResult)
t.Errorf("RangeSizes expected (%v), got (%v)", expectedResult, returnedResult)
}
}