Skip to content

Commit e82f7d4

Browse files
dinaldinal
authored
Merge pull request #511 from dinal/master
dev -> master
2 parents 20efcd8 + 1023611 commit e82f7d4

File tree

12 files changed

+190
-113
lines changed

12 files changed

+190
-113
lines changed

Diff for: ‎.github/workflows/ci.yaml

+132
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,132 @@
1+
name: CI
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- development
7+
- 'v[0-9]+.[0-9]+.x'
8+
9+
# Run CI also on push to master
10+
push:
11+
branches:
12+
- master
13+
14+
release:
15+
types:
16+
- created
17+
18+
jobs:
19+
test:
20+
name: Lint & test
21+
runs-on: ubuntu-latest
22+
env:
23+
TSDB_TEST_TABLE_PATH: TSDB_INTEGRATION_TESTS/${{ github.sha }}
24+
25+
steps:
26+
- name: Dump github context
27+
run: echo "$GITHUB_CONTEXT"
28+
env:
29+
GITHUB_CONTEXT: ${{ toJson(github) }}
30+
31+
- name: Dump runner context
32+
run: echo "$RUNNER_CONTEXT"
33+
env:
34+
RUNNER_CONTEXT: ${{ toJson(runner) }}
35+
36+
- name: Dump github ref
37+
run: echo "$GITHUB_REF"
38+
39+
- uses: actions/setup-go@v2
40+
with:
41+
go-version: "^1.14.0"
42+
43+
- uses: actions/cache@v2
44+
with:
45+
path: ~/go/pkg/mod
46+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
47+
restore-keys: |
48+
${{ runner.os }}-go-
49+
50+
- uses: actions/checkout@v2
51+
52+
- name: Lint
53+
run: make lint
54+
55+
- name: Test short
56+
run: make test
57+
58+
# Disabled until GH actions will have a solution to using secret on PR triggered workflows.
59+
# Until then, this runs in travis
60+
# - name: Test integration
61+
# run: make integration
62+
# env:
63+
# V3IO_TSDB_CONFIG: ${{ github.workspace }}/test/ci_v3io.yaml
64+
#
65+
# - name: Benchmark
66+
# run: make bench
67+
# env:
68+
# V3IO_TSDB_CONFIG: ${{ github.workspace }}/test/ci_v3io_bench.yaml
69+
# TSDB_BENCH_INGEST_CONFIG: ${{ github.workspace }}/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml
70+
71+
release:
72+
name: Release
73+
runs-on: ubuntu-latest
74+
if: github.event_name == 'release'
75+
needs:
76+
- test
77+
strategy:
78+
matrix:
79+
go-os:
80+
- linux
81+
- darwin
82+
- windows
83+
84+
steps:
85+
- name: Dump GitHub context
86+
run: echo "$GITHUB_CONTEXT"
87+
env:
88+
GITHUB_CONTEXT: ${{ toJson(github) }}
89+
90+
- name: Dump runner context
91+
run: echo "$RUNNER_CONTEXT"
92+
env:
93+
RUNNER_CONTEXT: ${{ toJson(runner) }}
94+
95+
- name: Dump github ref
96+
run: echo "$GITHUB_REF"
97+
98+
- name: Extract ref info
99+
id: release_info
100+
run: |
101+
echo ::set-output name=REF_BRANCH::${GITHUB_REF#refs/heads/}
102+
echo ::set-output name=REF_TAG::${GITHUB_REF#refs/tags/}
103+
104+
- name: Set TSDB_LABEL to release tag
105+
run: |
106+
echo "::set-env name=TSDB_LABEL::${{ steps.release_info.outputs.REF_TAG }}"
107+
108+
- uses: actions/checkout@v2
109+
110+
- uses: actions/setup-go@v2
111+
with:
112+
go-version: "^1.14.0"
113+
114+
- uses: actions/cache@v2
115+
with:
116+
path: ~/go/pkg/mod
117+
key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
118+
restore-keys: |
119+
${{ runner.os }}-go-
120+
121+
- name: Build binaries for ${{ matrix.go-os }}
122+
run: make bin
123+
env:
124+
GOPATH: ${{ github.workspace }}/go
125+
GOARCH: amd64
126+
GOOS: ${{ matrix.go-os }}
127+
128+
- name: Upload binaries
129+
uses: AButler/[email protected]
130+
with:
131+
files: ${{ github.workspace }}/go/bin/tsdbctl-*
132+
repo-token: ${{ secrets.GITHUB_TOKEN }}

Diff for: ‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,4 @@
55
*.sln.iml
66
.idea/
77

8+
tsdbctl-*

Diff for: ‎.travis.yml

+2-7
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,5 @@ env:
99
script:
1010
- make test
1111
- make lint
12-
- V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/travis_v3io.yaml" make integration
13-
- V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/travis_v3io_bench.yaml" TSDB_BENCH_INGEST_CONFIG="$TRAVIS_BUILD_DIR/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml" make bench
14-
15-
before_deploy:
16-
- GOOS=linux GOARCH=amd64 make build
17-
- GOOS=darwin GOARCH=amd64 make build
18-
- GOOS=windows GOARCH=amd64 make build
12+
- V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/test/ci_v3io.yaml" make integration
13+
- V3IO_TSDB_CONFIG="$TRAVIS_BUILD_DIR/test/ci_v3io_bench.yaml" TSDB_BENCH_INGEST_CONFIG="$TRAVIS_BUILD_DIR/test/benchmark/testdata/tsdb-bench-test-config-ci.yaml" make bench

Diff for: ‎Jenkinsfile

-59
Original file line numberDiff line numberDiff line change
@@ -9,54 +9,6 @@ git_deploy_user_token = "iguazio-prod-git-user-token"
99
git_deploy_user_private_key = "iguazio-prod-git-user-private-key"
1010

1111

12-
def build_v3io_tsdb(TAG_VERSION) {
13-
withCredentials([
14-
string(credentialsId: git_deploy_user_token, variable: 'GIT_TOKEN')
15-
]) {
16-
def git_project = 'v3io-tsdb'
17-
stage('prepare sources') {
18-
container('jnlp') {
19-
dir("${BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
20-
git(changelog: false, credentialsId: git_deploy_user_private_key, poll: false, url: "[email protected]:${git_project_user}/${git_project}.git")
21-
sh("git checkout ${TAG_VERSION}")
22-
}
23-
}
24-
}
25-
26-
stage("build ${git_project} binaries in dood") {
27-
container('golang') {
28-
dir("${BUILD_FOLDER}/src/github.com/${git_project_upstream_user}/${git_project}") {
29-
sh """
30-
mkdir -p /home/jenkins/go/bin
31-
GO111MODULE=on GOOS=linux GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin
32-
GO111MODULE=on GOOS=darwin GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin
33-
GO111MODULE=on GOOS=windows GOARCH=amd64 TRAVIS_TAG=${TAG_VERSION} make bin
34-
ls -la /go/bin
35-
mv /go/bin/* /home/jenkins/go/bin/
36-
ls -la /home/jenkins/go/bin/
37-
38-
"""
39-
}
40-
}
41-
}
42-
43-
stage('upload release assets') {
44-
container('jnlp') {
45-
RELEASE_ID = github.get_release_id(git_project, git_project_user, "${TAG_VERSION}", GIT_TOKEN)
46-
47-
github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-linux-amd64", RELEASE_ID, GIT_TOKEN)
48-
github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-darwin-amd64", RELEASE_ID, GIT_TOKEN)
49-
github.upload_asset(git_project, git_project_user, "tsdbctl-${TAG_VERSION}-windows-amd64", RELEASE_ID, GIT_TOKEN)
50-
withCredentials([
51-
string(credentialsId: pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[2], variable: 'PACKAGES_ARTIFACTORY_PASSWORD')
52-
]) {
53-
common.upload_file_to_artifactory(pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[0], pipelinex.PackagesRepo.ARTIFACTORY_IGUAZIO[1], PACKAGES_ARTIFACTORY_PASSWORD, "iguazio-devops/k8s", "tsdbctl-${TAG_VERSION}-linux-amd64")
54-
}
55-
}
56-
}
57-
}
58-
}
59-
6012
def build_nuclio(V3IO_TSDB_VERSION, internal_status="stable") {
6113
withCredentials([
6214
usernamePassword(credentialsId: git_deploy_user, passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USERNAME'),
@@ -413,17 +365,6 @@ podTemplate(label: "${git_project}-${label}", inheritFrom: "jnlp-docker-golang")
413365

414366
if (github.check_tag_expiration(git_project, git_project_user, MAIN_TAG_VERSION, GIT_TOKEN)) {
415367
parallel(
416-
'v3io-tsdb': {
417-
podTemplate(label: "v3io-tsdb-${label}", inheritFrom: "jnlp-docker-golang") {
418-
node("v3io-tsdb-${label}") {
419-
withCredentials([
420-
string(credentialsId: git_deploy_user_token, variable: 'GIT_TOKEN')
421-
]) {
422-
build_v3io_tsdb(MAIN_TAG_VERSION)
423-
}
424-
}
425-
}
426-
},
427368
'tsdb-nuclio': {
428369
podTemplate(label: "v3io-tsdb-nuclio-${label}", inheritFrom: "jnlp-docker-golang") {
429370
node("v3io-tsdb-nuclio-${label}") {

Diff for: ‎Makefile

+2-2
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@ ifeq ($(GIT_BRANCH),)
44
GIT_BRANCH="N/A"
55
endif
66

7-
ifneq ($(TRAVIS_TAG),)
8-
GIT_REVISION := $(TRAVIS_TAG)
7+
ifneq ($(TSDB_LABEL),)
8+
GIT_REVISION := $(TSDB_LABEL)
99
else
1010
GIT_REVISION := $(shell git describe --always)
1111
endif

Diff for: ‎README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
[![Build Status](https://travis-ci.org/v3io/v3io-tsdb.svg?branch=master)](https://travis-ci.org/v3io/v3io-tsdb)
1+
[![Travis Build Status](https://travis-ci.org/v3io/v3io-tsdb.svg?branch=master)](https://travis-ci.org/v3io/v3io-tsdb)
2+
[![GH Build Status](https://github.com/v3io/v3io-tsdb/workflows/CI/badge.svg)](https://github.com/v3io/v3io-tsdb/actions)
23

34
# V3IO-TSDB
45
Iguazio API lib for time-series DB access and Prometheus TSDB storage driver.

Diff for: ‎pkg/appender/appender.go

-4
Original file line numberDiff line numberDiff line change
@@ -77,10 +77,6 @@ func (m *MetricState) isReady() bool {
7777
return m.state == storeStateReady
7878
}
7979

80-
func (m *MetricState) isTimeInvalid(t int64) bool {
81-
return !((m.state == storeStateReady || m.state == storeStateUpdate) && t < m.store.maxTime-maxLateArrivalInterval)
82-
}
83-
8480
func (m *MetricState) getState() storeState {
8581
return m.state
8682
}

Diff for: ‎pkg/appender/ingest.go

+24-34
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,6 @@ func (mc *MetricsCache) metricFeed(index int) {
4949
go func() {
5050
inFlight := 0
5151
gotData := false
52-
gotCompletion := false
5352
potentialCompletion := false
5453
var completeChan chan int
5554

@@ -64,14 +63,13 @@ func (mc *MetricsCache) metricFeed(index int) {
6463

6564
// If data was sent and the queue is empty, mark as completion
6665
if length == 0 && gotData {
67-
if len(mc.asyncAppendChan) == 0 {
68-
gotCompletion = true
66+
switch len(mc.asyncAppendChan) {
67+
case 0:
68+
potentialCompletion = true
6969
if completeChan != nil {
7070
completeChan <- 0
71-
gotData = false
72-
potentialCompletion = false
7371
}
74-
} else if len(mc.asyncAppendChan) == 1 {
72+
case 1:
7573
potentialCompletion = true
7674
}
7775
}
@@ -84,41 +82,33 @@ func (mc *MetricsCache) metricFeed(index int) {
8482
if app.metric == nil {
8583
// Handle update completion requests (metric == nil)
8684
completeChan = app.resp
87-
length := mc.metricQueue.Length()
88-
if length == 0 && len(mc.asyncAppendChan) == 0 {
89-
if gotCompletion || (potentialCompletion && gotData) {
90-
completeChan <- 0
91-
gotCompletion = false
92-
gotData = false
93-
}
85+
if potentialCompletion {
86+
completeChan <- 0
9487
}
95-
potentialCompletion = false
9688
} else {
9789
potentialCompletion = false
9890
// Handle append requests (Add / AddFast)
9991
gotData = true
10092
metric := app.metric
10193
metric.Lock()
10294

103-
if metric.isTimeInvalid(app.t) {
104-
metric.store.Append(app.t, app.v)
105-
numPushed++
106-
dataQueued += metric.store.samplesQueueLength()
107-
108-
// If there are no in-flight requests, add the metric to the queue and update state
109-
if metric.isReady() || metric.getState() == storeStateInit {
110-
111-
if metric.getState() == storeStateInit {
112-
metric.setState(storeStatePreGet)
113-
}
114-
if metric.isReady() {
115-
metric.setState(storeStateUpdate)
116-
}
117-
118-
length := mc.metricQueue.Push(metric)
119-
if length < 2*mc.cfg.Workers {
120-
newMetrics++
121-
}
95+
metric.store.Append(app.t, app.v)
96+
numPushed++
97+
dataQueued += metric.store.samplesQueueLength()
98+
99+
// If there are no in-flight requests, add the metric to the queue and update state
100+
if metric.isReady() || metric.getState() == storeStateInit {
101+
102+
if metric.getState() == storeStateInit {
103+
metric.setState(storeStatePreGet)
104+
}
105+
if metric.isReady() {
106+
metric.setState(storeStateUpdate)
107+
}
108+
109+
length := mc.metricQueue.Push(metric)
110+
if length < 2*mc.cfg.Workers {
111+
newMetrics++
122112
}
123113
}
124114
metric.Unlock()
@@ -217,7 +207,7 @@ func (mc *MetricsCache) metricsUpdateLoop(index int) {
217207
}
218208

219209
// Notify the metric feeder when all in-flight tasks are done
220-
if mc.updatesInFlight == 0 && len(mc.asyncAppendChan) == 0 {
210+
if mc.updatesInFlight == 0 {
221211
mc.logger.Debug("Return to feed. Metric queue length: %d", mc.metricQueue.Length())
222212
mc.updatesComplete <- 0
223213
}

Diff for: ‎pkg/utils/timeutils.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -108,11 +108,11 @@ func Str2unixTime(timeString string) (int64, error) {
108108
if err != nil {
109109
return 0, errors.Wrap(err, "Invalid time string - not an RFC 3339 time format.")
110110
}
111-
return t.Unix() * 1000, nil
111+
return t.UnixNano() / int64(time.Millisecond), nil
112112
}
113113

114114
func CurrentTimeInMillis() int64 {
115-
return time.Now().Unix() * 1000
115+
return time.Now().UnixNano() / int64(time.Millisecond)
116116
}
117117

118118
func GetTimeFromRange(from, to, last, step string) (f int64, t int64, s int64, err error) {

0 commit comments

Comments
 (0)