op-deployer: Add forge autodiscovery (ethereum-optimism#17152)

* op-deployer: Add forge autodiscovery

* add limit reader

* cr updates

Author: mslipper

op-deployer/pkg/deployer/apply.go

@@ -7,6 +7,8 @@ import (
 	"math/big"
 	"strings"
 
+	"github.com/ethereum-optimism/optimism/op-service/ioutil"
+
 	"github.com/ethereum-optimism/optimism/devnet-sdk/proofs/prestate"
 	"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
 	"github.com/ethereum-optimism/optimism/op-chain-ops/script"
@@ -170,7 +172,7 @@ func ApplyPipeline(
 	}
 	st := opts.State
 
-	l1ArtifactsFS, err := artifacts.Download(ctx, intent.L1ContractsLocator, artifacts.BarProgressor(), opts.CacheDir)
+	l1ArtifactsFS, err := artifacts.Download(ctx, intent.L1ContractsLocator, ioutil.BarProgressor(), opts.CacheDir)
 	if err != nil {
 		return fmt.Errorf("failed to download L1 artifacts: %w", err)
 	}
@@ -179,7 +181,7 @@ func ApplyPipeline(
 	if intent.L1ContractsLocator.Equal(intent.L2ContractsLocator) {
 		l2ArtifactsFS = l1ArtifactsFS
 	} else {
-		l2Afs, err := artifacts.Download(ctx, intent.L2ContractsLocator, artifacts.BarProgressor(), opts.CacheDir)
+		l2Afs, err := artifacts.Download(ctx, intent.L2ContractsLocator, ioutil.BarProgressor(), opts.CacheDir)
 		if err != nil {
 			return fmt.Errorf("failed to download L2 artifacts: %w", err)
 		}

op-deployer/pkg/deployer/artifacts/download.go

@@ -8,14 +8,14 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
-	"io"
 	"io/fs"
-	"net/http"
 	"net/url"
 	"os"
 	"path"
 	"sync"
 
+	"github.com/ethereum-optimism/optimism/op-service/httputil"
+
 	"github.com/ethereum-optimism/optimism/op-service/ioutil"
 
 	"github.com/ethereum-optimism/optimism/op-chain-ops/foundry"
@@ -24,16 +24,16 @@ import (
 var ErrUnsupportedArtifactsScheme = errors.New("unsupported artifacts URL scheme")
 
 type Downloader interface {
-	Download(ctx context.Context, url string, progress DownloadProgressor, targetDir string) (string, error)
+	Download(ctx context.Context, url string, progress ioutil.Progressor, targetDir string) (string, error)
 }
 
 type Extractor interface {
 	Extract(src string, dest string) (string, error)
 }
 
-func Download(ctx context.Context, loc *Locator, progressor DownloadProgressor, targetDir string) (foundry.StatDirFs, error) {
+func Download(ctx context.Context, loc *Locator, progressor ioutil.Progressor, targetDir string) (foundry.StatDirFs, error) {
 	if progressor == nil {
-		progressor = NoopProgressor()
+		progressor = ioutil.NoopProgressor()
 	}
 
 	var err error
@@ -60,7 +60,7 @@ func Download(ctx context.Context, loc *Locator, progressor DownloadProgressor,
 	return artifactsFS.(foundry.StatDirFs), nil
 }
 
-func downloadHTTP(ctx context.Context, u *url.URL, progressor DownloadProgressor, checker integrityChecker, targetDir string) (fs.FS, error) {
+func downloadHTTP(ctx context.Context, u *url.URL, progressor ioutil.Progressor, checker integrityChecker, targetDir string) (fs.FS, error) {
 	cacher := &CachingDownloader{
 		d: new(HTTPDownloader),
 	}
@@ -84,38 +84,20 @@ func downloadHTTP(ctx context.Context, u *url.URL, progressor DownloadProgressor
 
 type HTTPDownloader struct{}
 
-func (d *HTTPDownloader) Download(ctx context.Context, url string, progress DownloadProgressor, targetDir string) (string, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return "", fmt.Errorf("failed to create request: %w", err)
-	}
-
-	res, err := http.DefaultClient.Do(req)
-	if err != nil {
-		return "", fmt.Errorf("failed to download artifacts: %w", err)
-	}
-	if res.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("failed to download artifacts: invalid status code %s", res.Status)
-	}
-	defer res.Body.Close()
-
+func (d *HTTPDownloader) Download(ctx context.Context, url string, progress ioutil.Progressor, targetDir string) (string, error) {
 	if err := os.MkdirAll(targetDir, 0755); err != nil {
 		return "", fmt.Errorf("failed to ensure cache directory '%s': %w", targetDir, err)
 	}
 	tmpFile, err := os.CreateTemp(targetDir, "op-deployer-artifacts-*")
 	if err != nil {
 		return "", fmt.Errorf("failed to create temporary file: %w", err)
 	}
-
-	pr := &progressReader{
-		r:        res.Body,
-		progress: progress,
-		total:    res.ContentLength,
+	downloader := &httputil.Downloader{
+		Progressor: progress,
 	}
-	if _, err := io.Copy(tmpFile, pr); err != nil {
-		return "", fmt.Errorf("failed to write to temporary file: %w", err)
+	if err := downloader.Download(ctx, url, tmpFile); err != nil {
+		return "", fmt.Errorf("failed to download: %w", err)
 	}
-
 	return tmpFile.Name(), nil
 }
 
@@ -124,7 +106,7 @@ type CachingDownloader struct {
 	mtx sync.Mutex
 }
 
-func (d *CachingDownloader) Download(ctx context.Context, url string, progress DownloadProgressor, targetDir string) (string, error) {
+func (d *CachingDownloader) Download(ctx context.Context, url string, progress ioutil.Progressor, targetDir string) (string, error) {
 	d.mtx.Lock()
 	defer d.mtx.Unlock()
 

op-deployer/pkg/deployer/artifacts/progress.go

@@ -144,7 +144,7 @@ func Implementations(ctx context.Context, cfg ImplementationsConfig) (opcm.Deplo
 
 	lgr := cfg.Logger
 
-	artifactsFS, err := artifacts.Download(ctx, cfg.ArtifactsLocator, artifacts.BarProgressor(), cfg.CacheDir)
+	artifactsFS, err := artifacts.Download(ctx, cfg.ArtifactsLocator, ioutil.BarProgressor(), cfg.CacheDir)
 	if err != nil {
 		return dio, fmt.Errorf("failed to download artifacts: %w", err)
 	}

op-deployer/pkg/deployer/bootstrap/implementations.go

@@ -114,7 +114,7 @@ func Proxy(ctx context.Context, cfg ProxyConfig) (opcm.DeployProxyOutput, error)
 	}
 
 	lgr := cfg.Logger
-	artifactsFS, err := artifacts.Download(ctx, cfg.ArtifactsLocator, artifacts.BarProgressor(), cfg.CacheDir)
+	artifactsFS, err := artifacts.Download(ctx, cfg.ArtifactsLocator, ioutil.BarProgressor(), cfg.CacheDir)
 	if err != nil {
 		return dpo, fmt.Errorf("failed to download artifacts: %w", err)
 	}

op-deployer/pkg/deployer/bootstrap/proxy.go

@@ -151,7 +151,7 @@ func Superchain(ctx context.Context, cfg SuperchainConfig) (opcm.DeploySuperchai
 
 	lgr := cfg.Logger
 	cacheDir := cfg.CacheDir
-	artifactsFS, err := artifacts.Download(ctx, cfg.ArtifactsLocator, artifacts.BarProgressor(), cacheDir)
+	artifactsFS, err := artifacts.Download(ctx, cfg.ArtifactsLocator, ioutil.BarProgressor(), cacheDir)
 	if err != nil {
 		return dso, fmt.Errorf("failed to download artifacts: %w", err)
 	}