data sink: add incremental mode

This commit is contained in:
Chris Lu
2021-02-28 16:19:03 -08:00
parent 4ff2c5c4c9
commit 678c54d705
9 changed files with 60 additions and 15 deletions

View File

@@ -22,6 +22,7 @@ type GcsSink struct {
bucket string
dir string
filerSource *source.FilerSource
isIncremental bool
}
func init() {
@@ -36,7 +37,12 @@ func (g *GcsSink) GetSinkToDirectory() string {
return g.dir
}
func (g *GcsSink) IsIncremental() bool {
return g.isIncremental
}
func (g *GcsSink) Initialize(configuration util.Configuration, prefix string) error {
g.isIncremental = configuration.GetBool(prefix+"is_incremental")
return g.initialize(
configuration.GetString(prefix+"google_application_credentials"),
configuration.GetString(prefix+"bucket"),