Update dependencies

This commit is contained in:
Manuel de Brito Fontes 2017-10-06 17:33:32 -03:00
parent bf5616c65b
commit d6d374b28d
13962 changed files with 48226 additions and 3618880 deletions

View file

@ -1 +0,0 @@
Forked from etcd 2.2 release branch to support migration from 3.0 WAL to 2.2 WAL format

View file

@ -1,52 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = [
"fileutil_test.go",
"lock_test.go",
"preallocate_test.go",
"purge_test.go",
],
library = ":go_default_library",
)
go_library(
name = "go_default_library",
srcs = [
"fileutil.go",
"lock_unix.go",
"perallocate_unsupported.go",
"purge.go",
] + select({
"@io_bazel_rules_go//go/platform:linux_amd64": [
"preallocate.go",
],
"@io_bazel_rules_go//go/platform:windows_amd64": [
"lock_windows.go",
],
"//conditions:default": [],
}),
deps = ["//vendor/github.com/coreos/pkg/capnslog:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View file

@ -1,57 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"io/ioutil"
"os"
"path"
"sort"
"github.com/coreos/pkg/capnslog"
)
const (
privateFileMode = 0600
)
var (
plog = capnslog.NewPackageLogger("github.com/coreos/etcd/pkg", "fileutil")
)
// IsDirWriteable checks if dir is writable by writing and removing a file
// to dir. It returns nil if dir is writable.
func IsDirWriteable(dir string) error {
f := path.Join(dir, ".touch")
if err := ioutil.WriteFile(f, []byte(""), privateFileMode); err != nil {
return err
}
return os.Remove(f)
}
// ReadDir returns the filenames in the given directory in sorted order.
func ReadDir(dirpath string) ([]string, error) {
dir, err := os.Open(dirpath)
if err != nil {
return nil, err
}
defer dir.Close()
names, err := dir.Readdirnames(-1)
if err != nil {
return nil, err
}
sort.Strings(names)
return names, nil
}

View file

@ -1,67 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"io/ioutil"
"os"
"path/filepath"
"reflect"
"testing"
)
func TestIsDirWriteable(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected ioutil.TempDir error: %v", err)
}
defer os.RemoveAll(tmpdir)
if err := IsDirWriteable(tmpdir); err != nil {
t.Fatalf("unexpected IsDirWriteable error: %v", err)
}
if err := os.Chmod(tmpdir, 0444); err != nil {
t.Fatalf("unexpected os.Chmod error: %v", err)
}
if err := IsDirWriteable(tmpdir); err == nil {
t.Fatalf("expected IsDirWriteable to error")
}
}
func TestReadDir(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
defer os.RemoveAll(tmpdir)
if err != nil {
t.Fatalf("unexpected ioutil.TempDir error: %v", err)
}
files := []string{"def", "abc", "xyz", "ghi"}
for _, f := range files {
var fh *os.File
fh, err = os.Create(filepath.Join(tmpdir, f))
if err != nil {
t.Fatalf("error creating file: %v", err)
}
if err := fh.Close(); err != nil {
t.Fatalf("error closing file: %v", err)
}
}
fs, err := ReadDir(tmpdir)
if err != nil {
t.Fatalf("error calling ReadDir: %v", err)
}
wfs := []string{"abc", "def", "ghi", "xyz"}
if !reflect.DeepEqual(fs, wfs) {
t.Fatalf("ReadDir: got %v, want %v", fs, wfs)
}
}

View file

@ -1,90 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"errors"
"os"
"syscall"
"time"
)
var (
ErrLocked = errors.New("file already locked")
)
type Lock interface {
Name() string
TryLock() error
Lock() error
Unlock() error
Destroy() error
}
type lock struct {
fname string
file *os.File
}
func (l *lock) Name() string {
return l.fname
}
// TryLock acquires exclusivity on the lock without blocking
func (l *lock) TryLock() error {
err := os.Chmod(l.fname, syscall.DMEXCL|0600)
if err != nil {
return err
}
f, err := os.Open(l.fname)
if err != nil {
return ErrLocked
}
l.file = f
return nil
}
// Lock acquires exclusivity on the lock with blocking
func (l *lock) Lock() error {
err := os.Chmod(l.fname, syscall.DMEXCL|0600)
if err != nil {
return err
}
for {
f, err := os.Open(l.fname)
if err == nil {
l.file = f
return nil
}
time.Sleep(10 * time.Millisecond)
}
}
// Unlock unlocks the lock
func (l *lock) Unlock() error {
return l.file.Close()
}
func (l *lock) Destroy() error {
return nil
}
func NewLock(file string) (Lock, error) {
l := &lock{fname: file}
return l, nil
}

View file

@ -1,98 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build solaris
package fileutil
import (
"errors"
"os"
"syscall"
)
var (
ErrLocked = errors.New("file already locked")
)
type Lock interface {
Name() string
TryLock() error
Lock() error
Unlock() error
Destroy() error
}
type lock struct {
fd int
file *os.File
}
func (l *lock) Name() string {
return l.file.Name()
}
// TryLock acquires exclusivity on the lock without blocking
func (l *lock) TryLock() error {
var lock syscall.Flock_t
lock.Start = 0
lock.Len = 0
lock.Pid = 0
lock.Type = syscall.F_WRLCK
lock.Whence = 0
lock.Pid = 0
err := syscall.FcntlFlock(uintptr(l.fd), syscall.F_SETLK, &lock)
if err != nil && err == syscall.EAGAIN {
return ErrLocked
}
return err
}
// Lock acquires exclusivity on the lock without blocking
func (l *lock) Lock() error {
var lock syscall.Flock_t
lock.Start = 0
lock.Len = 0
lock.Type = syscall.F_WRLCK
lock.Whence = 0
lock.Pid = 0
return syscall.FcntlFlock(uintptr(l.fd), syscall.F_SETLK, &lock)
}
// Unlock unlocks the lock
func (l *lock) Unlock() error {
var lock syscall.Flock_t
lock.Start = 0
lock.Len = 0
lock.Type = syscall.F_UNLCK
lock.Whence = 0
err := syscall.FcntlFlock(uintptr(l.fd), syscall.F_SETLK, &lock)
if err != nil && err == syscall.EAGAIN {
return ErrLocked
}
return err
}
func (l *lock) Destroy() error {
return l.file.Close()
}
func NewLock(file string) (Lock, error) {
f, err := os.OpenFile(file, os.O_WRONLY, 0600)
if err != nil {
return nil, err
}
l := &lock{int(f.Fd()), f}
return l, nil
}

View file

@ -1,96 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"io/ioutil"
"os"
"testing"
"time"
)
func TestLockAndUnlock(t *testing.T) {
f, err := ioutil.TempFile("", "lock")
if err != nil {
t.Fatal(err)
}
f.Close()
defer func() {
err := os.Remove(f.Name())
if err != nil {
t.Fatal(err)
}
}()
// lock the file
l, err := NewLock(f.Name())
if err != nil {
t.Fatal(err)
}
defer l.Destroy()
err = l.Lock()
if err != nil {
t.Fatal(err)
}
// try lock a locked file
dupl, err := NewLock(f.Name())
if err != nil {
t.Fatal(err)
}
err = dupl.TryLock()
if err != ErrLocked {
t.Errorf("err = %v, want %v", err, ErrLocked)
}
// unlock the file
err = l.Unlock()
if err != nil {
t.Fatal(err)
}
// try lock the unlocked file
err = dupl.TryLock()
if err != nil {
t.Errorf("err = %v, want %v", err, nil)
}
defer dupl.Destroy()
// blocking on locked file
locked := make(chan struct{}, 1)
go func() {
l.Lock()
locked <- struct{}{}
}()
select {
case <-locked:
t.Error("unexpected unblocking")
case <-time.After(10 * time.Millisecond):
}
// unlock
err = dupl.Unlock()
if err != nil {
t.Fatal(err)
}
// the previously blocked routine should be unblocked
select {
case <-locked:
case <-time.After(20 * time.Millisecond):
t.Error("unexpected blocking")
}
}

View file

@ -1,76 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !windows,!plan9,!solaris
package fileutil
import (
"errors"
"os"
"syscall"
)
var (
ErrLocked = errors.New("file already locked")
)
type Lock interface {
Name() string
TryLock() error
Lock() error
Unlock() error
Destroy() error
}
type lock struct {
fd int
file *os.File
}
func (l *lock) Name() string {
return l.file.Name()
}
// TryLock acquires exclusivity on the lock without blocking
func (l *lock) TryLock() error {
err := syscall.Flock(l.fd, syscall.LOCK_EX|syscall.LOCK_NB)
if err != nil && err == syscall.EWOULDBLOCK {
return ErrLocked
}
return err
}
// Lock acquires exclusivity on the lock without blocking
func (l *lock) Lock() error {
return syscall.Flock(l.fd, syscall.LOCK_EX)
}
// Unlock unlocks the lock
func (l *lock) Unlock() error {
return syscall.Flock(l.fd, syscall.LOCK_UN)
}
func (l *lock) Destroy() error {
return l.file.Close()
}
func NewLock(file string) (Lock, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
l := &lock{int(f.Fd()), f}
return l, nil
}

View file

@ -1,71 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build windows
package fileutil
import (
"errors"
"os"
)
var (
ErrLocked = errors.New("file already locked")
)
type Lock interface {
Name() string
TryLock() error
Lock() error
Unlock() error
Destroy() error
}
type lock struct {
fd int
file *os.File
}
func (l *lock) Name() string {
return l.file.Name()
}
// TryLock acquires exclusivity on the lock without blocking
func (l *lock) TryLock() error {
return nil
}
// Lock acquires exclusivity on the lock without blocking
func (l *lock) Lock() error {
return nil
}
// Unlock unlocks the lock
func (l *lock) Unlock() error {
return nil
}
func (l *lock) Destroy() error {
return l.file.Close()
}
func NewLock(file string) (Lock, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
l := &lock{int(f.Fd()), f}
return l, nil
}

View file

@ -1,28 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !linux
package fileutil
import "os"
// Preallocate tries to allocate the space for given
// file. This operation is only supported on linux by a
// few filesystems (btrfs, ext4, etc.).
// If the operation is unsupported, no error will be returned.
// Otherwise, the error encountered will be returned.
func Preallocate(f *os.File, sizeInBytes int) error {
return nil
}

View file

@ -1,42 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build linux
package fileutil
import (
"os"
"syscall"
)
// Preallocate tries to allocate the space for given
// file. This operation is only supported on linux by a
// few filesystems (btrfs, ext4, etc.).
// If the operation is unsupported, no error will be returned.
// Otherwise, the error encountered will be returned.
func Preallocate(f *os.File, sizeInBytes int) error {
// use mode = 1 to keep size
// see FALLOC_FL_KEEP_SIZE
err := syscall.Fallocate(int(f.Fd()), 1, 0, int64(sizeInBytes))
if err != nil {
errno, ok := err.(syscall.Errno)
// treat not support as nil error
if ok && errno == syscall.ENOTSUP {
return nil
}
return err
}
return nil
}

View file

@ -1,53 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"io/ioutil"
"os"
"runtime"
"testing"
)
func TestPreallocate(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skipf("skip testPreallocate, OS = %s", runtime.GOOS)
}
p, err := ioutil.TempDir(os.TempDir(), "preallocateTest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(p)
f, err := ioutil.TempFile(p, "")
if err != nil {
t.Fatal(err)
}
size := 64 * 1000
err = Preallocate(f, size)
if err != nil {
t.Fatal(err)
}
stat, err := f.Stat()
if err != nil {
t.Fatal(err)
}
if stat.Size() != 0 {
t.Errorf("size = %d, want %d", stat.Size(), 0)
}
}

View file

@ -1,80 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"os"
"path"
"sort"
"strings"
"time"
)
func PurgeFile(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) <-chan error {
errC := make(chan error, 1)
go func() {
for {
fnames, err := ReadDir(dirname)
if err != nil {
errC <- err
return
}
newfnames := make([]string, 0)
for _, fname := range fnames {
if strings.HasSuffix(fname, suffix) {
newfnames = append(newfnames, fname)
}
}
sort.Strings(newfnames)
for len(newfnames) > int(max) {
f := path.Join(dirname, newfnames[0])
l, err := NewLock(f)
if err != nil {
errC <- err
return
}
err = l.TryLock()
if err != nil {
break
}
err = os.Remove(f)
if err != nil {
errC <- err
return
}
err = l.Unlock()
if err != nil {
plog.Errorf("error unlocking %s when purging file (%v)", l.Name(), err)
errC <- err
return
}
err = l.Destroy()
if err != nil {
plog.Errorf("error destroying lock %s when purging file (%v)", l.Name(), err)
errC <- err
return
}
plog.Infof("purged file %s successfully", f)
newfnames = newfnames[1:]
}
select {
case <-time.After(interval):
case <-stop:
return
}
}
}()
return errC
}

View file

@ -1,132 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"fmt"
"io/ioutil"
"os"
"path"
"reflect"
"testing"
"time"
)
func TestPurgeFile(t *testing.T) {
dir, err := ioutil.TempDir("", "purgefile")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
for i := 0; i < 5; i++ {
_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
if err != nil {
t.Fatal(err)
}
}
stop := make(chan struct{})
errch := PurgeFile(dir, "test", 3, time.Millisecond, stop)
for i := 5; i < 10; i++ {
_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
if err != nil {
t.Fatal(err)
}
time.Sleep(10 * time.Millisecond)
}
fnames, err := ReadDir(dir)
if err != nil {
t.Fatal(err)
}
wnames := []string{"7.test", "8.test", "9.test"}
if !reflect.DeepEqual(fnames, wnames) {
t.Errorf("filenames = %v, want %v", fnames, wnames)
}
select {
case err := <-errch:
t.Errorf("unexpected purge error %v", err)
case <-time.After(time.Millisecond):
}
close(stop)
}
func TestPurgeFileHoldingLock(t *testing.T) {
dir, err := ioutil.TempDir("", "purgefile")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
for i := 0; i < 10; i++ {
_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
if err != nil {
t.Fatal(err)
}
}
// create a purge barrier at 5
l, err := NewLock(path.Join(dir, fmt.Sprintf("%d.test", 5)))
err = l.Lock()
if err != nil {
t.Fatal(err)
}
stop := make(chan struct{})
errch := PurgeFile(dir, "test", 3, time.Millisecond, stop)
time.Sleep(20 * time.Millisecond)
fnames, err := ReadDir(dir)
if err != nil {
t.Fatal(err)
}
wnames := []string{"5.test", "6.test", "7.test", "8.test", "9.test"}
if !reflect.DeepEqual(fnames, wnames) {
t.Errorf("filenames = %v, want %v", fnames, wnames)
}
select {
case err := <-errch:
t.Errorf("unexpected purge error %v", err)
case <-time.After(time.Millisecond):
}
// remove the purge barrier
err = l.Unlock()
if err != nil {
t.Fatal(err)
}
err = l.Destroy()
if err != nil {
t.Fatal(err)
}
time.Sleep(20 * time.Millisecond)
fnames, err = ReadDir(dir)
if err != nil {
t.Fatal(err)
}
wnames = []string{"7.test", "8.test", "9.test"}
if !reflect.DeepEqual(fnames, wnames) {
t.Errorf("filenames = %v, want %v", fnames, wnames)
}
select {
case err := <-errch:
t.Errorf("unexpected purge error %v", err)
case <-time.After(time.Millisecond):
}
close(stop)
}

View file

@ -1,48 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"decoder.go",
"doc.go",
"encoder.go",
"metrics.go",
"multi_readcloser.go",
"repair.go",
"util.go",
"wal.go",
],
deps = [
"//third_party/forked/etcd221/pkg/fileutil:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/crc:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/pbutil:go_default_library",
"//vendor/github.com/coreos/etcd/raft:go_default_library",
"//vendor/github.com/coreos/etcd/raft/raftpb:go_default_library",
"//vendor/github.com/coreos/etcd/wal/walpb:go_default_library",
"//vendor/github.com/coreos/pkg/capnslog:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//third_party/forked/etcd221/wal/walpb:all-srcs",
],
tags = ["automanaged"],
)

View file

@ -1,103 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"bufio"
"encoding/binary"
"hash"
"io"
"sync"
"github.com/coreos/etcd/pkg/crc"
"github.com/coreos/etcd/pkg/pbutil"
"github.com/coreos/etcd/raft/raftpb"
"github.com/coreos/etcd/wal/walpb"
)
type decoder struct {
mu sync.Mutex
br *bufio.Reader
c io.Closer
crc hash.Hash32
}
func newDecoder(rc io.ReadCloser) *decoder {
return &decoder{
br: bufio.NewReader(rc),
c: rc,
crc: crc.New(0, crcTable),
}
}
func (d *decoder) decode(rec *walpb.Record) error {
d.mu.Lock()
defer d.mu.Unlock()
rec.Reset()
l, err := readInt64(d.br)
if err != nil {
return err
}
data := make([]byte, l)
if _, err = io.ReadFull(d.br, data); err != nil {
// ReadFull returns io.EOF only if no bytes were read
// the decoder should treat this as an ErrUnexpectedEOF instead.
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
}
if err := rec.Unmarshal(data); err != nil {
return err
}
// skip crc checking if the record type is crcType
if rec.Type == crcType {
return nil
}
d.crc.Write(rec.Data)
return rec.Validate(d.crc.Sum32())
}
func (d *decoder) updateCRC(prevCrc uint32) {
d.crc = crc.New(prevCrc, crcTable)
}
func (d *decoder) lastCRC() uint32 {
return d.crc.Sum32()
}
func (d *decoder) close() error {
return d.c.Close()
}
func mustUnmarshalEntry(d []byte) raftpb.Entry {
var e raftpb.Entry
pbutil.MustUnmarshal(&e, d)
return e
}
func mustUnmarshalState(d []byte) raftpb.HardState {
var s raftpb.HardState
pbutil.MustUnmarshal(&s, d)
return s
}
func readInt64(r io.Reader) (int64, error) {
var n int64
err := binary.Read(r, binary.LittleEndian, &n)
return n, err
}

View file

@ -1,68 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package wal provides an implementation of a write ahead log that is used by
etcd.
A WAL is created at a particular directory and is made up of a number of
segmented WAL files. Inside of each file the raft state and entries are appended
to it with the Save method:
metadata := []byte{}
w, err := wal.Create("/var/lib/etcd", metadata)
...
err := w.Save(s, ents)
After saving an raft snapshot to disk, SaveSnapshot method should be called to
record it. So WAL can match with the saved snapshot when restarting.
err := w.SaveSnapshot(walpb.Snapshot{Index: 10, Term: 2})
When a user has finished using a WAL it must be closed:
w.Close()
WAL files are placed inside of the directory in the following format:
$seq-$index.wal
The first WAL file to be created will be 0000000000000000-0000000000000000.wal
indicating an initial sequence of 0 and an initial raft index of 0. The first
entry written to WAL MUST have raft index 0.
WAL will cuts its current wal files if its size exceeds 8MB. This will increment an internal
sequence number and cause a new file to be created. If the last raft index saved
was 0x20 and this is the first time cut has been called on this WAL then the sequence will
increment from 0x0 to 0x1. The new file will be: 0000000000000001-0000000000000021.wal.
If a second cut issues 0x10 entries with incremental index later then the file will be called:
0000000000000002-0000000000000031.wal.
At a later time a WAL can be opened at a particular snapshot. If there is no
snapshot, an empty snapshot should be passed in.
w, err := wal.Open("/var/lib/etcd", walpb.Snapshot{Index: 10, Term: 2})
...
The snapshot must have been written to the WAL.
Additional items cannot be Saved to this WAL until all of the items from the given
snapshot to the end of the WAL are read first:
metadata, state, ents, err := w.ReadAll()
This will give you the metadata, the last raft.State and the slice of
raft.Entry items in the log.
*/
package wal

View file

@ -1,89 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"bufio"
"encoding/binary"
"hash"
"io"
"sync"
"github.com/coreos/etcd/pkg/crc"
"github.com/coreos/etcd/wal/walpb"
)
type encoder struct {
mu sync.Mutex
bw *bufio.Writer
crc hash.Hash32
buf []byte
uint64buf []byte
}
func newEncoder(w io.Writer, prevCrc uint32) *encoder {
return &encoder{
bw: bufio.NewWriter(w),
crc: crc.New(prevCrc, crcTable),
// 1MB buffer
buf: make([]byte, 1024*1024),
uint64buf: make([]byte, 8),
}
}
func (e *encoder) encode(rec *walpb.Record) error {
e.mu.Lock()
defer e.mu.Unlock()
e.crc.Write(rec.Data)
rec.Crc = e.crc.Sum32()
var (
data []byte
err error
n int
)
if rec.Size() > len(e.buf) {
data, err = rec.Marshal()
if err != nil {
return err
}
} else {
n, err = rec.MarshalTo(e.buf)
if err != nil {
return err
}
data = e.buf[:n]
}
if err := writeInt64(e.bw, int64(len(data)), e.uint64buf); err != nil {
return err
}
_, err = e.bw.Write(data)
return err
}
func (e *encoder) flush() error {
e.mu.Lock()
defer e.mu.Unlock()
return e.bw.Flush()
}
func writeInt64(w io.Writer, n int64, buf []byte) error {
// http://golang.org/src/encoding/binary/binary.go
binary.LittleEndian.PutUint64(buf, uint64(n))
_, err := w.Write(buf)
return err
}

View file

@ -1,37 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import "github.com/prometheus/client_golang/prometheus"
var (
syncDurations = prometheus.NewSummary(prometheus.SummaryOpts{
Namespace: "etcd",
Subsystem: "wal",
Name: "fsync_durations_microseconds",
Help: "The latency distributions of fsync called by wal.",
})
lastIndexSaved = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "etcd",
Subsystem: "wal",
Name: "last_index_saved",
Help: "The index of the last entry saved by wal.",
})
)
func init() {
prometheus.MustRegister(syncDurations)
prometheus.MustRegister(lastIndexSaved)
}

View file

@ -1,45 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import "io"
type multiReadCloser struct {
closers []io.Closer
reader io.Reader
}
func (mc *multiReadCloser) Close() error {
var err error
for i := range mc.closers {
err = mc.closers[i].Close()
}
return err
}
func (mc *multiReadCloser) Read(p []byte) (int, error) {
return mc.reader.Read(p)
}
func MultiReadCloser(readClosers ...io.ReadCloser) io.ReadCloser {
cs := make([]io.Closer, len(readClosers))
rs := make([]io.Reader, len(readClosers))
for i := range readClosers {
cs[i] = readClosers[i]
rs[i] = readClosers[i]
}
r := io.MultiReader(rs...)
return &multiReadCloser{cs, r}
}

View file

@ -1,107 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"io"
"os"
"path"
"k8s.io/kubernetes/third_party/forked/etcd221/pkg/fileutil"
"github.com/coreos/etcd/wal/walpb"
)
// Repair tries to repair the unexpectedEOF error in the
// last wal file by truncating.
func Repair(dirpath string) bool {
f, err := openLast(dirpath)
if err != nil {
return false
}
defer f.Close()
n := 0
rec := &walpb.Record{}
decoder := newDecoder(f)
defer decoder.close()
for {
err := decoder.decode(rec)
switch err {
case nil:
n += 8 + rec.Size()
// update crc of the decoder when necessary
switch rec.Type {
case crcType:
crc := decoder.crc.Sum32()
// current crc of decoder must match the crc of the record.
// do no need to match 0 crc, since the decoder is a new one at this case.
if crc != 0 && rec.Validate(crc) != nil {
return false
}
decoder.updateCRC(rec.Crc)
}
continue
case io.EOF:
return true
case io.ErrUnexpectedEOF:
plog.Noticef("repairing %v", f.Name())
bf, bferr := os.Create(f.Name() + ".broken")
if bferr != nil {
plog.Errorf("could not repair %v, failed to create backup file", f.Name())
return false
}
defer bf.Close()
if _, err = f.Seek(0, os.SEEK_SET); err != nil {
plog.Errorf("could not repair %v, failed to read file", f.Name())
return false
}
if _, err = io.Copy(bf, f); err != nil {
plog.Errorf("could not repair %v, failed to copy file", f.Name())
return false
}
if err = f.Truncate(int64(n)); err != nil {
plog.Errorf("could not repair %v, failed to truncate file", f.Name())
return false
}
if err = f.Sync(); err != nil {
plog.Errorf("could not repair %v, failed to sync file", f.Name())
return false
}
return true
default:
plog.Errorf("could not repair error (%v)", err)
return false
}
}
}
// openLast opens the last wal file for read and write.
func openLast(dirpath string) (*os.File, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
last := path.Join(dirpath, names[len(names)-1])
return os.OpenFile(last, os.O_RDWR, 0)
}

View file

@ -1,93 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"errors"
"fmt"
"strings"
"k8s.io/kubernetes/third_party/forked/etcd221/pkg/fileutil"
)
var (
badWalName = errors.New("bad wal name")
)
func Exist(dirpath string) bool {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return false
}
return len(names) != 0
}
// searchIndex returns the last array index of names whose raft index section is
// equal to or smaller than the given index.
// The given names MUST be sorted.
func searchIndex(names []string, index uint64) (int, bool) {
for i := len(names) - 1; i >= 0; i-- {
name := names[i]
_, curIndex, err := parseWalName(name)
if err != nil {
plog.Panicf("parse correct name should never fail: %v", err)
}
if index >= curIndex {
return i, true
}
}
return -1, false
}
// names should have been sorted based on sequence number.
// isValidSeq checks whether seq increases continuously.
func isValidSeq(names []string) bool {
var lastSeq uint64
for _, name := range names {
curSeq, _, err := parseWalName(name)
if err != nil {
plog.Panicf("parse correct name should never fail: %v", err)
}
if lastSeq != 0 && lastSeq != curSeq-1 {
return false
}
lastSeq = curSeq
}
return true
}
func checkWalNames(names []string) []string {
wnames := make([]string, 0)
for _, name := range names {
if _, _, err := parseWalName(name); err != nil {
plog.Warningf("ignored file %v in wal", name)
continue
}
wnames = append(wnames, name)
}
return wnames
}
func parseWalName(str string) (seq, index uint64, err error) {
if !strings.HasSuffix(str, ".wal") {
return 0, 0, badWalName
}
_, err = fmt.Sscanf(str, "%016x-%016x.wal", &seq, &index)
return seq, index, err
}
func walName(seq, index uint64) string {
return fmt.Sprintf("%016x-%016x.wal", seq, index)
}

View file

@ -1,548 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"errors"
"fmt"
"hash/crc32"
"io"
"os"
"path"
"reflect"
"sync"
"time"
"k8s.io/kubernetes/third_party/forked/etcd221/pkg/fileutil"
"github.com/coreos/etcd/pkg/pbutil"
"github.com/coreos/etcd/raft"
"github.com/coreos/etcd/raft/raftpb"
"github.com/coreos/etcd/wal/walpb"
"github.com/coreos/pkg/capnslog"
)
const (
metadataType int64 = iota + 1
entryType
stateType
crcType
snapshotType
// the owner can make/remove files inside the directory
privateDirMode = 0700
// the expected size of each wal segment file.
// the actual size might be bigger than it.
segmentSizeBytes = 64 * 1000 * 1000 // 64MB
)
var (
plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "wal")
ErrMetadataConflict = errors.New("wal: conflicting metadata found")
ErrFileNotFound = errors.New("wal: file not found")
ErrCRCMismatch = errors.New("wal: crc mismatch")
ErrSnapshotMismatch = errors.New("wal: snapshot mismatch")
ErrSnapshotNotFound = errors.New("wal: snapshot not found")
crcTable = crc32.MakeTable(crc32.Castagnoli)
)
// WAL is a logical repersentation of the stable storage.
// WAL is either in read mode or append mode but not both.
// A newly created WAL is in append mode, and ready for appending records.
// A just opened WAL is in read mode, and ready for reading records.
// The WAL will be ready for appending after reading out all the previous records.
type WAL struct {
dir string // the living directory of the underlay files
metadata []byte // metadata recorded at the head of each WAL
state raftpb.HardState // hardstate recorded at the head of WAL
start walpb.Snapshot // snapshot to start reading
decoder *decoder // decoder to decode records
mu sync.Mutex
f *os.File // underlay file opened for appending, sync
seq uint64 // sequence of the wal file currently used for writes
enti uint64 // index of the last entry saved to the wal
encoder *encoder // encoder to encode records
locks []fileutil.Lock // the file locks the WAL is holding (the name is increasing)
}
// Create creates a WAL ready for appending records. The given metadata is
// recorded at the head of each WAL file, and can be retrieved with ReadAll.
func Create(dirpath string, metadata []byte) (*WAL, error) {
if Exist(dirpath) {
return nil, os.ErrExist
}
if err := os.MkdirAll(dirpath, privateDirMode); err != nil {
return nil, err
}
p := path.Join(dirpath, walName(0, 0))
f, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
if err != nil {
return nil, err
}
l, err := fileutil.NewLock(f.Name())
if err != nil {
return nil, err
}
err = l.Lock()
if err != nil {
return nil, err
}
w := &WAL{
dir: dirpath,
metadata: metadata,
seq: 0,
f: f,
encoder: newEncoder(f, 0),
}
w.locks = append(w.locks, l)
if err := w.saveCrc(0); err != nil {
return nil, err
}
if err := w.encoder.encode(&walpb.Record{Type: metadataType, Data: metadata}); err != nil {
return nil, err
}
if err = w.SaveSnapshot(walpb.Snapshot{}); err != nil {
return nil, err
}
return w, nil
}
// Open opens the WAL at the given snap.
// The snap SHOULD have been previously saved to the WAL, or the following
// ReadAll will fail.
// The returned WAL is ready to read and the first record will be the one after
// the given snap. The WAL cannot be appended to before reading out all of its
// previous records.
func Open(dirpath string, snap walpb.Snapshot) (*WAL, error) {
return openAtIndex(dirpath, snap, true)
}
// OpenForRead only opens the wal files for read.
// Write on a read only wal panics.
func OpenForRead(dirpath string, snap walpb.Snapshot) (*WAL, error) {
return openAtIndex(dirpath, snap, false)
}
func openAtIndex(dirpath string, snap walpb.Snapshot, write bool) (*WAL, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
nameIndex, ok := searchIndex(names, snap.Index)
if !ok || !isValidSeq(names[nameIndex:]) {
return nil, ErrFileNotFound
}
// open the wal files for reading
rcs := make([]io.ReadCloser, 0)
ls := make([]fileutil.Lock, 0)
for _, name := range names[nameIndex:] {
f, err := os.Open(path.Join(dirpath, name))
if err != nil {
return nil, err
}
l, err := fileutil.NewLock(f.Name())
if err != nil {
return nil, err
}
err = l.TryLock()
if err != nil {
if write {
return nil, err
}
}
rcs = append(rcs, f)
ls = append(ls, l)
}
rc := MultiReadCloser(rcs...)
// create a WAL ready for reading
w := &WAL{
dir: dirpath,
start: snap,
decoder: newDecoder(rc),
locks: ls,
}
if write {
// open the lastest wal file for appending
seq, _, err := parseWalName(names[len(names)-1])
if err != nil {
rc.Close()
return nil, err
}
last := path.Join(dirpath, names[len(names)-1])
f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
if err != nil {
rc.Close()
return nil, err
}
err = fileutil.Preallocate(f, segmentSizeBytes)
if err != nil {
rc.Close()
plog.Errorf("failed to allocate space when creating new wal file (%v)", err)
return nil, err
}
w.f = f
w.seq = seq
}
return w, nil
}
// ReadAll reads out records of the current WAL.
// If opened in write mode, it must read out all records until EOF. Or an error
// will be returned.
// If opened in read mode, it will try to read all records if possible.
// If it cannot read out the expected snap, it will return ErrSnapshotNotFound.
// If loaded snap doesn't match with the expected one, it will return
// all the records and error ErrSnapshotMismatch.
// TODO: detect not-last-snap error.
// TODO: maybe loose the checking of match.
// After ReadAll, the WAL will be ready for appending new records.
func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb.Entry, err error) {
w.mu.Lock()
defer w.mu.Unlock()
rec := &walpb.Record{}
decoder := w.decoder
var match bool
for err = decoder.decode(rec); err == nil; err = decoder.decode(rec) {
switch rec.Type {
case entryType:
e := mustUnmarshalEntry(rec.Data)
if e.Index > w.start.Index {
ents = append(ents[:e.Index-w.start.Index-1], e)
}
w.enti = e.Index
case stateType:
state = mustUnmarshalState(rec.Data)
case metadataType:
if metadata != nil && !reflect.DeepEqual(metadata, rec.Data) {
state.Reset()
return nil, state, nil, ErrMetadataConflict
}
metadata = rec.Data
case crcType:
crc := decoder.crc.Sum32()
// current crc of decoder must match the crc of the record.
// do no need to match 0 crc, since the decoder is a new one at this case.
if crc != 0 && rec.Validate(crc) != nil {
state.Reset()
return nil, state, nil, ErrCRCMismatch
}
decoder.updateCRC(rec.Crc)
case snapshotType:
var snap walpb.Snapshot
pbutil.MustUnmarshal(&snap, rec.Data)
if snap.Index == w.start.Index {
if snap.Term != w.start.Term {
state.Reset()
return nil, state, nil, ErrSnapshotMismatch
}
match = true
}
default:
state.Reset()
return nil, state, nil, fmt.Errorf("unexpected block type %d", rec.Type)
}
}
switch w.f {
case nil:
// We do not have to read out all entries in read mode.
// The last record maybe a partial written one, so
// ErrunexpectedEOF might be returned.
if err != io.EOF && err != io.ErrUnexpectedEOF {
state.Reset()
return nil, state, nil, err
}
default:
// We must read all of the entries if WAL is opened in write mode.
if err != io.EOF {
state.Reset()
return nil, state, nil, err
}
}
err = nil
if !match {
err = ErrSnapshotNotFound
}
// close decoder, disable reading
w.decoder.close()
w.start = walpb.Snapshot{}
w.metadata = metadata
if w.f != nil {
// create encoder (chain crc with the decoder), enable appending
w.encoder = newEncoder(w.f, w.decoder.lastCRC())
w.decoder = nil
lastIndexSaved.Set(float64(w.enti))
}
return metadata, state, ents, err
}
// cut closes current file written and creates a new one ready to append.
// cut first creates a temp wal file and writes necessary headers into it.
// Then cut atomtically rename temp wal file to a wal file.
func (w *WAL) cut() error {
// close old wal file
if err := w.sync(); err != nil {
return err
}
if err := w.f.Close(); err != nil {
return err
}
fpath := path.Join(w.dir, walName(w.seq+1, w.enti+1))
ftpath := fpath + ".tmp"
// create a temp wal file with name sequence + 1, or tuncate the existing one
ft, err := os.OpenFile(ftpath, os.O_WRONLY|os.O_APPEND|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
// update writer and save the previous crc
w.f = ft
prevCrc := w.encoder.crc.Sum32()
w.encoder = newEncoder(w.f, prevCrc)
if err := w.saveCrc(prevCrc); err != nil {
return err
}
if err := w.encoder.encode(&walpb.Record{Type: metadataType, Data: w.metadata}); err != nil {
return err
}
if err := w.saveState(&w.state); err != nil {
return err
}
// close temp wal file
if err := w.sync(); err != nil {
return err
}
if err := w.f.Close(); err != nil {
return err
}
// atomically move temp wal file to wal file
if err := os.Rename(ftpath, fpath); err != nil {
return err
}
// open the wal file and update writer again
f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
return err
}
err = fileutil.Preallocate(f, segmentSizeBytes)
if err != nil {
plog.Errorf("failed to allocate space when creating new wal file (%v)", err)
return err
}
w.f = f
prevCrc = w.encoder.crc.Sum32()
w.encoder = newEncoder(w.f, prevCrc)
// lock the new wal file
l, err := fileutil.NewLock(f.Name())
if err != nil {
return err
}
err = l.Lock()
if err != nil {
return err
}
w.locks = append(w.locks, l)
// increase the wal seq
w.seq++
plog.Infof("segmented wal file %v is created", fpath)
return nil
}
func (w *WAL) sync() error {
if w.encoder != nil {
if err := w.encoder.flush(); err != nil {
return err
}
}
start := time.Now()
err := w.f.Sync()
syncDurations.Observe(float64(time.Since(start).Nanoseconds() / int64(time.Microsecond)))
return err
}
// ReleaseLockTo releases the locks, which has smaller index than the given index
// except the largest one among them.
// For example, if WAL is holding lock 1,2,3,4,5,6, ReleaseLockTo(4) will release
// lock 1,2 but keep 3. ReleaseLockTo(5) will release 1,2,3 but keep 4.
func (w *WAL) ReleaseLockTo(index uint64) error {
w.mu.Lock()
defer w.mu.Unlock()
var smaller int
found := false
for i, l := range w.locks {
_, lockIndex, err := parseWalName(path.Base(l.Name()))
if err != nil {
return err
}
if lockIndex >= index {
smaller = i - 1
found = true
break
}
}
// if no lock index is greater than the release index, we can
// release lock upto the last one(excluding).
if !found && len(w.locks) != 0 {
smaller = len(w.locks) - 1
}
if smaller <= 0 {
return nil
}
for i := 0; i < smaller; i++ {
w.locks[i].Unlock()
w.locks[i].Destroy()
}
w.locks = w.locks[smaller:]
return nil
}
func (w *WAL) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.f != nil {
if err := w.sync(); err != nil {
return err
}
if err := w.f.Close(); err != nil {
return err
}
}
for _, l := range w.locks {
err := l.Unlock()
if err != nil {
plog.Errorf("failed to unlock during closing wal: %s", err)
}
err = l.Destroy()
if err != nil {
plog.Errorf("failed to destroy lock during closing wal: %s", err)
}
}
return nil
}
func (w *WAL) saveEntry(e *raftpb.Entry) error {
// TODO: add MustMarshalTo to reduce one allocation.
b := pbutil.MustMarshal(e)
rec := &walpb.Record{Type: entryType, Data: b}
if err := w.encoder.encode(rec); err != nil {
return err
}
w.enti = e.Index
lastIndexSaved.Set(float64(w.enti))
return nil
}
func (w *WAL) saveState(s *raftpb.HardState) error {
if raft.IsEmptyHardState(*s) {
return nil
}
w.state = *s
b := pbutil.MustMarshal(s)
rec := &walpb.Record{Type: stateType, Data: b}
return w.encoder.encode(rec)
}
func (w *WAL) Save(st raftpb.HardState, ents []raftpb.Entry) error {
w.mu.Lock()
defer w.mu.Unlock()
// short cut, do not call sync
if raft.IsEmptyHardState(st) && len(ents) == 0 {
return nil
}
// TODO(xiangli): no more reference operator
for i := range ents {
if err := w.saveEntry(&ents[i]); err != nil {
return err
}
}
if err := w.saveState(&st); err != nil {
return err
}
fstat, err := w.f.Stat()
if err != nil {
return err
}
if fstat.Size() < segmentSizeBytes {
return w.sync()
}
// TODO: add a test for this code path when refactoring the tests
return w.cut()
}
func (w *WAL) SaveSnapshot(e walpb.Snapshot) error {
w.mu.Lock()
defer w.mu.Unlock()
b := pbutil.MustMarshal(&e)
rec := &walpb.Record{Type: snapshotType, Data: b}
if err := w.encoder.encode(rec); err != nil {
return err
}
// update enti only when snapshot is ahead of last index
if w.enti < e.Index {
w.enti = e.Index
}
lastIndexSaved.Set(float64(w.enti))
return w.sync()
}
func (w *WAL) saveCrc(prevCrc uint32) error {
return w.encoder.encode(&walpb.Record{Type: crcType, Crc: prevCrc})
}

View file

@ -1,36 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"record.go",
"record.pb.go",
],
deps = ["//vendor/github.com/gogo/protobuf/proto:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
filegroup(
name = "go_default_library_protos",
srcs = ["record.proto"],
visibility = ["//visibility:public"],
)

View file

@ -1,29 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package walpb
import "errors"
var (
ErrCRCMismatch = errors.New("walpb: crc mismatch")
)
func (rec *Record) Validate(crc uint32) error {
if rec.Crc == crc {
return nil
}
rec.Reset()
return ErrCRCMismatch
}

View file

@ -1,447 +0,0 @@
// Code generated by protoc-gen-gogo.
// source: record.proto
// DO NOT EDIT!
/*
Package walpb is a generated protocol buffer package.
It is generated from these files:
record.proto
It has these top-level messages:
Record
Snapshot
*/
package walpb
import proto "github.com/gogo/protobuf/proto"
import math "math"
// discarding unused import gogoproto "github.com/coreos/etcd/Godeps/_workspace/src/gogoproto"
import io "io"
import fmt "fmt"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = math.Inf
type Record struct {
Type int64 `protobuf:"varint,1,opt,name=type" json:"type"`
Crc uint32 `protobuf:"varint,2,opt,name=crc" json:"crc"`
Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Record) Reset() { *m = Record{} }
func (m *Record) String() string { return proto.CompactTextString(m) }
func (*Record) ProtoMessage() {}
type Snapshot struct {
Index uint64 `protobuf:"varint,1,opt,name=index" json:"index"`
Term uint64 `protobuf:"varint,2,opt,name=term" json:"term"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Snapshot) Reset() { *m = Snapshot{} }
func (m *Snapshot) String() string { return proto.CompactTextString(m) }
func (*Snapshot) ProtoMessage() {}
func (m *Record) Marshal() (data []byte, err error) {
size := m.Size()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *Record) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
data[i] = 0x8
i++
i = encodeVarintRecord(data, i, uint64(m.Type))
data[i] = 0x10
i++
i = encodeVarintRecord(data, i, uint64(m.Crc))
if m.Data != nil {
data[i] = 0x1a
i++
i = encodeVarintRecord(data, i, uint64(len(m.Data)))
i += copy(data[i:], m.Data)
}
if m.XXX_unrecognized != nil {
i += copy(data[i:], m.XXX_unrecognized)
}
return i, nil
}
func (m *Snapshot) Marshal() (data []byte, err error) {
size := m.Size()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *Snapshot) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
data[i] = 0x8
i++
i = encodeVarintRecord(data, i, uint64(m.Index))
data[i] = 0x10
i++
i = encodeVarintRecord(data, i, uint64(m.Term))
if m.XXX_unrecognized != nil {
i += copy(data[i:], m.XXX_unrecognized)
}
return i, nil
}
func encodeFixed64Record(data []byte, offset int, v uint64) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
data[offset+4] = uint8(v >> 32)
data[offset+5] = uint8(v >> 40)
data[offset+6] = uint8(v >> 48)
data[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Record(data []byte, offset int, v uint32) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintRecord(data []byte, offset int, v uint64) int {
for v >= 1<<7 {
data[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
data[offset] = uint8(v)
return offset + 1
}
func (m *Record) Size() (n int) {
var l int
_ = l
n += 1 + sovRecord(uint64(m.Type))
n += 1 + sovRecord(uint64(m.Crc))
if m.Data != nil {
l = len(m.Data)
n += 1 + l + sovRecord(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *Snapshot) Size() (n int) {
var l int
_ = l
n += 1 + sovRecord(uint64(m.Index))
n += 1 + sovRecord(uint64(m.Term))
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovRecord(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozRecord(x uint64) (n int) {
return sovRecord(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *Record) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType)
}
m.Type = 0
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Type |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Crc", wireType)
}
m.Crc = 0
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Crc |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
byteLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthRecord
}
postIndex := iNdEx + byteLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Data = append([]byte{}, data[iNdEx:postIndex]...)
iNdEx = postIndex
default:
var sizeOfWire int
for {
sizeOfWire++
wire >>= 7
if wire == 0 {
break
}
}
iNdEx -= sizeOfWire
skippy, err := skipRecord(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthRecord
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
return nil
}
func (m *Snapshot) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
m.Index = 0
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Index |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Term", wireType)
}
m.Term = 0
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Term |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
var sizeOfWire int
for {
sizeOfWire++
wire >>= 7
if wire == 0 {
break
}
}
iNdEx -= sizeOfWire
skippy, err := skipRecord(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthRecord
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
return nil
}
func skipRecord(data []byte) (n int, err error) {
l := len(data)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for {
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if data[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthRecord
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipRecord(data[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthRecord = fmt.Errorf("proto: negative length found during unmarshaling")
)

View file

@ -1,20 +0,0 @@
syntax = "proto2";
package walpb;
import "gogoproto/gogo.proto";
option (gogoproto.marshaler_all) = true;
option (gogoproto.sizer_all) = true;
option (gogoproto.unmarshaler_all) = true;
option (gogoproto.goproto_getters_all) = false;
message Record {
optional int64 type = 1 [(gogoproto.nullable) = false];
optional uint32 crc = 2 [(gogoproto.nullable) = false];
optional bytes data = 3;
}
message Snapshot {
optional uint64 index = 1 [(gogoproto.nullable) = false];
optional uint64 term = 2 [(gogoproto.nullable) = false];
}

View file

@ -1 +0,0 @@
Forked from etcd 2.3 release branch to support migration from 3.0 WAL to 2.3 WAL format

View file

@ -1,55 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = [
"fileutil_test.go",
"lock_test.go",
"preallocate_test.go",
"purge_test.go",
],
library = ":go_default_library",
)
go_library(
name = "go_default_library",
srcs = [
"fileutil.go",
"lock.go",
"lock_unix.go",
"perallocate_unsupported.go",
"purge.go",
"sync.go",
] + select({
"@io_bazel_rules_go//go/platform:linux_amd64": [
"preallocate.go",
"sync_linux.go",
],
"@io_bazel_rules_go//go/platform:windows_amd64": [
"lock_windows.go",
],
"//conditions:default": [],
}),
deps = ["//vendor/github.com/coreos/pkg/capnslog:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View file

@ -1,75 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package fileutil implements utility functions related to files and paths.
package fileutil
import (
"io/ioutil"
"os"
"path"
"sort"
"github.com/coreos/pkg/capnslog"
)
const (
privateFileMode = 0600
// owner can make/remove files inside the directory
privateDirMode = 0700
)
var (
plog = capnslog.NewPackageLogger("github.com/coreos/etcd/pkg", "fileutil")
)
// IsDirWriteable checks if dir is writable by writing and removing a file
// to dir. It returns nil if dir is writable.
func IsDirWriteable(dir string) error {
f := path.Join(dir, ".touch")
if err := ioutil.WriteFile(f, []byte(""), privateFileMode); err != nil {
return err
}
return os.Remove(f)
}
// ReadDir returns the filenames in the given directory in sorted order.
func ReadDir(dirpath string) ([]string, error) {
dir, err := os.Open(dirpath)
if err != nil {
return nil, err
}
defer dir.Close()
names, err := dir.Readdirnames(-1)
if err != nil {
return nil, err
}
sort.Strings(names)
return names, nil
}
// TouchDirAll is similar to os.MkdirAll. It creates directories with 0700 permission if any directory
// does not exists. TouchDirAll also ensures the given directory is writable.
func TouchDirAll(dir string) error {
err := os.MkdirAll(dir, privateDirMode)
if err != nil && err != os.ErrExist {
return err
}
return IsDirWriteable(dir)
}
func Exist(name string) bool {
_, err := os.Stat(name)
return err == nil
}

View file

@ -1,96 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"io/ioutil"
"os"
"os/user"
"path/filepath"
"reflect"
"testing"
)
func TestIsDirWriteable(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
if err != nil {
t.Fatalf("unexpected ioutil.TempDir error: %v", err)
}
defer os.RemoveAll(tmpdir)
if err = IsDirWriteable(tmpdir); err != nil {
t.Fatalf("unexpected IsDirWriteable error: %v", err)
}
if err = os.Chmod(tmpdir, 0444); err != nil {
t.Fatalf("unexpected os.Chmod error: %v", err)
}
me, err := user.Current()
if err != nil {
// err can be non-nil when cross compiled
// http://stackoverflow.com/questions/20609415/cross-compiling-user-current-not-implemented-on-linux-amd64
t.Skipf("failed to get current user: %v", err)
}
if me.Name == "root" || me.Name == "Administrator" {
// ideally we should check CAP_DAC_OVERRIDE.
// but it does not matter for tests.
t.Skipf("running as a superuser")
}
if err := IsDirWriteable(tmpdir); err == nil {
t.Fatalf("expected IsDirWriteable to error")
}
}
func TestReadDir(t *testing.T) {
tmpdir, err := ioutil.TempDir("", "")
defer os.RemoveAll(tmpdir)
if err != nil {
t.Fatalf("unexpected ioutil.TempDir error: %v", err)
}
files := []string{"def", "abc", "xyz", "ghi"}
for _, f := range files {
var fh *os.File
fh, err = os.Create(filepath.Join(tmpdir, f))
if err != nil {
t.Fatalf("error creating file: %v", err)
}
if err = fh.Close(); err != nil {
t.Fatalf("error closing file: %v", err)
}
}
fs, err := ReadDir(tmpdir)
if err != nil {
t.Fatalf("error calling ReadDir: %v", err)
}
wfs := []string{"abc", "def", "ghi", "xyz"}
if !reflect.DeepEqual(fs, wfs) {
t.Fatalf("ReadDir: got %v, want %v", fs, wfs)
}
}
func TestExist(t *testing.T) {
f, err := ioutil.TempFile(os.TempDir(), "fileutil")
if err != nil {
t.Fatal(err)
}
f.Close()
if g := Exist(f.Name()); g != true {
t.Errorf("exist = %v, want true", g)
}
os.Remove(f.Name())
if g := Exist(f.Name()); g != false {
t.Errorf("exist = %v, want false", g)
}
}

View file

@ -1,29 +0,0 @@
// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
type Lock interface {
// Name returns the name of the file.
Name() string
// TryLock acquires exclusivity on the lock without blocking.
TryLock() error
// Lock acquires exclusivity on the lock.
Lock() error
// Unlock unlocks the lock.
Unlock() error
// Destroy should be called after Unlock to clean up
// the resources.
Destroy() error
}

View file

@ -1,79 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"errors"
"os"
"syscall"
"time"
)
var (
ErrLocked = errors.New("file already locked")
)
type lock struct {
fname string
file *os.File
}
func (l *lock) Name() string {
return l.fname
}
func (l *lock) TryLock() error {
err := os.Chmod(l.fname, syscall.DMEXCL|0600)
if err != nil {
return err
}
f, err := os.Open(l.fname)
if err != nil {
return ErrLocked
}
l.file = f
return nil
}
func (l *lock) Lock() error {
err := os.Chmod(l.fname, syscall.DMEXCL|0600)
if err != nil {
return err
}
for {
f, err := os.Open(l.fname)
if err == nil {
l.file = f
return nil
}
time.Sleep(10 * time.Millisecond)
}
}
func (l *lock) Unlock() error {
return l.file.Close()
}
func (l *lock) Destroy() error {
return nil
}
func NewLock(file string) (Lock, error) {
l := &lock{fname: file}
return l, nil
}

View file

@ -1,87 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build solaris
package fileutil
import (
"errors"
"os"
"syscall"
)
var (
ErrLocked = errors.New("file already locked")
)
type lock struct {
fd int
file *os.File
}
func (l *lock) Name() string {
return l.file.Name()
}
func (l *lock) TryLock() error {
var lock syscall.Flock_t
lock.Start = 0
lock.Len = 0
lock.Pid = 0
lock.Type = syscall.F_WRLCK
lock.Whence = 0
lock.Pid = 0
err := syscall.FcntlFlock(uintptr(l.fd), syscall.F_SETLK, &lock)
if err != nil && err == syscall.EAGAIN {
return ErrLocked
}
return err
}
func (l *lock) Lock() error {
var lock syscall.Flock_t
lock.Start = 0
lock.Len = 0
lock.Type = syscall.F_WRLCK
lock.Whence = 0
lock.Pid = 0
return syscall.FcntlFlock(uintptr(l.fd), syscall.F_SETLK, &lock)
}
func (l *lock) Unlock() error {
var lock syscall.Flock_t
lock.Start = 0
lock.Len = 0
lock.Type = syscall.F_UNLCK
lock.Whence = 0
err := syscall.FcntlFlock(uintptr(l.fd), syscall.F_SETLK, &lock)
if err != nil && err == syscall.EAGAIN {
return ErrLocked
}
return err
}
func (l *lock) Destroy() error {
return l.file.Close()
}
func NewLock(file string) (Lock, error) {
f, err := os.OpenFile(file, os.O_WRONLY, 0600)
if err != nil {
return nil, err
}
l := &lock{int(f.Fd()), f}
return l, nil
}

View file

@ -1,96 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"io/ioutil"
"os"
"testing"
"time"
)
func TestLockAndUnlock(t *testing.T) {
f, err := ioutil.TempFile("", "lock")
if err != nil {
t.Fatal(err)
}
f.Close()
defer func() {
err = os.Remove(f.Name())
if err != nil {
t.Fatal(err)
}
}()
// lock the file
l, err := NewLock(f.Name())
if err != nil {
t.Fatal(err)
}
defer l.Destroy()
err = l.Lock()
if err != nil {
t.Fatal(err)
}
// try lock a locked file
dupl, err := NewLock(f.Name())
if err != nil {
t.Fatal(err)
}
err = dupl.TryLock()
if err != ErrLocked {
t.Errorf("err = %v, want %v", err, ErrLocked)
}
// unlock the file
err = l.Unlock()
if err != nil {
t.Fatal(err)
}
// try lock the unlocked file
err = dupl.TryLock()
if err != nil {
t.Errorf("err = %v, want %v", err, nil)
}
defer dupl.Destroy()
// blocking on locked file
locked := make(chan struct{}, 1)
go func() {
l.Lock()
locked <- struct{}{}
}()
select {
case <-locked:
t.Error("unexpected unblocking")
case <-time.After(100 * time.Millisecond):
}
// unlock
err = dupl.Unlock()
if err != nil {
t.Fatal(err)
}
// the previously blocked routine should be unblocked
select {
case <-locked:
case <-time.After(1 * time.Second):
t.Error("unexpected blocking")
}
}

View file

@ -1,65 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !windows,!plan9,!solaris
package fileutil
import (
"errors"
"os"
"syscall"
)
var (
ErrLocked = errors.New("file already locked")
)
type lock struct {
fd int
file *os.File
}
func (l *lock) Name() string {
return l.file.Name()
}
func (l *lock) TryLock() error {
err := syscall.Flock(l.fd, syscall.LOCK_EX|syscall.LOCK_NB)
if err != nil && err == syscall.EWOULDBLOCK {
return ErrLocked
}
return err
}
func (l *lock) Lock() error {
return syscall.Flock(l.fd, syscall.LOCK_EX)
}
func (l *lock) Unlock() error {
return syscall.Flock(l.fd, syscall.LOCK_UN)
}
func (l *lock) Destroy() error {
return l.file.Close()
}
func NewLock(file string) (Lock, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
l := &lock{int(f.Fd()), f}
return l, nil
}

View file

@ -1,60 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build windows
package fileutil
import (
"errors"
"os"
)
var (
ErrLocked = errors.New("file already locked")
)
type lock struct {
fd int
file *os.File
}
func (l *lock) Name() string {
return l.file.Name()
}
func (l *lock) TryLock() error {
return nil
}
func (l *lock) Lock() error {
return nil
}
func (l *lock) Unlock() error {
return nil
}
func (l *lock) Destroy() error {
return l.file.Close()
}
func NewLock(file string) (Lock, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
l := &lock{int(f.Fd()), f}
return l, nil
}

View file

@ -1,28 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !linux
package fileutil
import "os"
// Preallocate tries to allocate the space for given
// file. This operation is only supported on linux by a
// few filesystems (btrfs, ext4, etc.).
// If the operation is unsupported, no error will be returned.
// Otherwise, the error encountered will be returned.
func Preallocate(f *os.File, sizeInBytes int) error {
return nil
}

View file

@ -1,42 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build linux
package fileutil
import (
"os"
"syscall"
)
// Preallocate tries to allocate the space for given
// file. This operation is only supported on linux by a
// few filesystems (btrfs, ext4, etc.).
// If the operation is unsupported, no error will be returned.
// Otherwise, the error encountered will be returned.
func Preallocate(f *os.File, sizeInBytes int) error {
// use mode = 1 to keep size
// see FALLOC_FL_KEEP_SIZE
err := syscall.Fallocate(int(f.Fd()), 1, 0, int64(sizeInBytes))
if err != nil {
errno, ok := err.(syscall.Errno)
// treat not support as nil error
if ok && errno == syscall.ENOTSUP {
return nil
}
return err
}
return nil
}

View file

@ -1,53 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"io/ioutil"
"os"
"runtime"
"testing"
)
func TestPreallocate(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skipf("skip testPreallocate, OS = %s", runtime.GOOS)
}
p, err := ioutil.TempDir(os.TempDir(), "preallocateTest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(p)
f, err := ioutil.TempFile(p, "")
if err != nil {
t.Fatal(err)
}
size := 64 * 1000
err = Preallocate(f, size)
if err != nil {
t.Fatal(err)
}
stat, err := f.Stat()
if err != nil {
t.Fatal(err)
}
if stat.Size() != 0 {
t.Errorf("size = %d, want %d", stat.Size(), 0)
}
}

View file

@ -1,80 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"os"
"path"
"sort"
"strings"
"time"
)
func PurgeFile(dirname string, suffix string, max uint, interval time.Duration, stop <-chan struct{}) <-chan error {
errC := make(chan error, 1)
go func() {
for {
fnames, err := ReadDir(dirname)
if err != nil {
errC <- err
return
}
newfnames := make([]string, 0)
for _, fname := range fnames {
if strings.HasSuffix(fname, suffix) {
newfnames = append(newfnames, fname)
}
}
sort.Strings(newfnames)
for len(newfnames) > int(max) {
f := path.Join(dirname, newfnames[0])
l, err := NewLock(f)
if err != nil {
errC <- err
return
}
err = l.TryLock()
if err != nil {
break
}
err = os.Remove(f)
if err != nil {
errC <- err
return
}
err = l.Unlock()
if err != nil {
plog.Errorf("error unlocking %s when purging file (%v)", l.Name(), err)
errC <- err
return
}
err = l.Destroy()
if err != nil {
plog.Errorf("error destroying lock %s when purging file (%v)", l.Name(), err)
errC <- err
return
}
plog.Infof("purged file %s successfully", f)
newfnames = newfnames[1:]
}
select {
case <-time.After(interval):
case <-stop:
return
}
}
}()
return errC
}

View file

@ -1,161 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fileutil
import (
"fmt"
"io/ioutil"
"os"
"path"
"reflect"
"testing"
"time"
)
func TestPurgeFile(t *testing.T) {
dir, err := ioutil.TempDir("", "purgefile")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
for i := 0; i < 5; i++ {
_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
if err != nil {
t.Fatal(err)
}
}
stop := make(chan struct{})
// keep at most 3 most recent files
errch := PurgeFile(dir, "test", 3, time.Millisecond, stop)
// create 5 more files
for i := 5; i < 10; i++ {
_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
if err != nil {
t.Fatal(err)
}
time.Sleep(10 * time.Millisecond)
}
// purge routine should purge 7 out of 10 files and only keep the
// 3 most recent ones.
// wait for purging for at most 100ms.
var fnames []string
for i := 0; i < 10; i++ {
fnames, err = ReadDir(dir)
if err != nil {
t.Fatal(err)
}
if len(fnames) <= 3 {
break
}
time.Sleep(10 * time.Millisecond)
}
wnames := []string{"7.test", "8.test", "9.test"}
if !reflect.DeepEqual(fnames, wnames) {
t.Errorf("filenames = %v, want %v", fnames, wnames)
}
// no error should be reported from purge routine
select {
case err := <-errch:
t.Errorf("unexpected purge error %v", err)
case <-time.After(time.Millisecond):
}
close(stop)
}
func TestPurgeFileHoldingLock(t *testing.T) {
dir, err := ioutil.TempDir("", "purgefile")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(dir)
for i := 0; i < 10; i++ {
_, err = os.Create(path.Join(dir, fmt.Sprintf("%d.test", i)))
if err != nil {
t.Fatal(err)
}
}
// create a purge barrier at 5
l, err := NewLock(path.Join(dir, fmt.Sprintf("%d.test", 5)))
err = l.Lock()
if err != nil {
t.Fatal(err)
}
stop := make(chan struct{})
errch := PurgeFile(dir, "test", 3, time.Millisecond, stop)
var fnames []string
for i := 0; i < 10; i++ {
fnames, err = ReadDir(dir)
if err != nil {
t.Fatal(err)
}
if len(fnames) <= 5 {
break
}
time.Sleep(10 * time.Millisecond)
}
wnames := []string{"5.test", "6.test", "7.test", "8.test", "9.test"}
if !reflect.DeepEqual(fnames, wnames) {
t.Errorf("filenames = %v, want %v", fnames, wnames)
}
select {
case err = <-errch:
t.Errorf("unexpected purge error %v", err)
case <-time.After(time.Millisecond):
}
// remove the purge barrier
err = l.Unlock()
if err != nil {
t.Fatal(err)
}
err = l.Destroy()
if err != nil {
t.Fatal(err)
}
for i := 0; i < 10; i++ {
fnames, err = ReadDir(dir)
if err != nil {
t.Fatal(err)
}
if len(fnames) <= 3 {
break
}
time.Sleep(10 * time.Millisecond)
}
wnames = []string{"7.test", "8.test", "9.test"}
if !reflect.DeepEqual(fnames, wnames) {
t.Errorf("filenames = %v, want %v", fnames, wnames)
}
select {
case err := <-errch:
t.Errorf("unexpected purge error %v", err)
case <-time.After(time.Millisecond):
}
close(stop)
}

View file

@ -1,26 +0,0 @@
// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !linux
package fileutil
import "os"
// Fdatasync is similar to fsync(), but does not flush modified metadata
// unless that metadata is needed in order to allow a subsequent data retrieval
// to be correctly handled.
func Fdatasync(f *os.File) error {
return f.Sync()
}

View file

@ -1,29 +0,0 @@
// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build linux
package fileutil
import (
"os"
"syscall"
)
// Fdatasync is similar to fsync(), but does not flush modified metadata
// unless that metadata is needed in order to allow a subsequent data retrieval
// to be correctly handled.
func Fdatasync(f *os.File) error {
return syscall.Fdatasync(int(f.Fd()))
}

View file

@ -1,47 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"decoder.go",
"doc.go",
"encoder.go",
"metrics.go",
"multi_readcloser.go",
"repair.go",
"util.go",
"wal.go",
],
deps = [
"//third_party/forked/etcd237/pkg/fileutil:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/crc:go_default_library",
"//vendor/github.com/coreos/etcd/pkg/pbutil:go_default_library",
"//vendor/github.com/coreos/etcd/raft/raftpb:go_default_library",
"//vendor/github.com/coreos/etcd/wal/walpb:go_default_library",
"//vendor/github.com/coreos/pkg/capnslog:go_default_library",
"//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//third_party/forked/etcd237/wal/walpb:all-srcs",
],
tags = ["automanaged"],
)

View file

@ -1,103 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"bufio"
"encoding/binary"
"hash"
"io"
"sync"
"github.com/coreos/etcd/pkg/crc"
"github.com/coreos/etcd/pkg/pbutil"
"github.com/coreos/etcd/raft/raftpb"
"github.com/coreos/etcd/wal/walpb"
)
type decoder struct {
mu sync.Mutex
br *bufio.Reader
c io.Closer
crc hash.Hash32
}
func newDecoder(rc io.ReadCloser) *decoder {
return &decoder{
br: bufio.NewReader(rc),
c: rc,
crc: crc.New(0, crcTable),
}
}
func (d *decoder) decode(rec *walpb.Record) error {
d.mu.Lock()
defer d.mu.Unlock()
rec.Reset()
l, err := readInt64(d.br)
if err != nil {
return err
}
data := make([]byte, l)
if _, err = io.ReadFull(d.br, data); err != nil {
// ReadFull returns io.EOF only if no bytes were read
// the decoder should treat this as an ErrUnexpectedEOF instead.
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
return err
}
if err := rec.Unmarshal(data); err != nil {
return err
}
// skip crc checking if the record type is crcType
if rec.Type == crcType {
return nil
}
d.crc.Write(rec.Data)
return rec.Validate(d.crc.Sum32())
}
func (d *decoder) updateCRC(prevCrc uint32) {
d.crc = crc.New(prevCrc, crcTable)
}
func (d *decoder) lastCRC() uint32 {
return d.crc.Sum32()
}
func (d *decoder) close() error {
return d.c.Close()
}
func mustUnmarshalEntry(d []byte) raftpb.Entry {
var e raftpb.Entry
pbutil.MustUnmarshal(&e, d)
return e
}
func mustUnmarshalState(d []byte) raftpb.HardState {
var s raftpb.HardState
pbutil.MustUnmarshal(&s, d)
return s
}
func readInt64(r io.Reader) (int64, error) {
var n int64
err := binary.Read(r, binary.LittleEndian, &n)
return n, err
}

View file

@ -1,68 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
Package wal provides an implementation of a write ahead log that is used by
etcd.
A WAL is created at a particular directory and is made up of a number of
segmented WAL files. Inside of each file the raft state and entries are appended
to it with the Save method:
metadata := []byte{}
w, err := wal.Create("/var/lib/etcd", metadata)
...
err := w.Save(s, ents)
After saving an raft snapshot to disk, SaveSnapshot method should be called to
record it. So WAL can match with the saved snapshot when restarting.
err := w.SaveSnapshot(walpb.Snapshot{Index: 10, Term: 2})
When a user has finished using a WAL it must be closed:
w.Close()
WAL files are placed inside of the directory in the following format:
$seq-$index.wal
The first WAL file to be created will be 0000000000000000-0000000000000000.wal
indicating an initial sequence of 0 and an initial raft index of 0. The first
entry written to WAL MUST have raft index 0.
WAL will cuts its current wal files if its size exceeds 8MB. This will increment an internal
sequence number and cause a new file to be created. If the last raft index saved
was 0x20 and this is the first time cut has been called on this WAL then the sequence will
increment from 0x0 to 0x1. The new file will be: 0000000000000001-0000000000000021.wal.
If a second cut issues 0x10 entries with incremental index later then the file will be called:
0000000000000002-0000000000000031.wal.
At a later time a WAL can be opened at a particular snapshot. If there is no
snapshot, an empty snapshot should be passed in.
w, err := wal.Open("/var/lib/etcd", walpb.Snapshot{Index: 10, Term: 2})
...
The snapshot must have been written to the WAL.
Additional items cannot be Saved to this WAL until all of the items from the given
snapshot to the end of the WAL are read first:
metadata, state, ents, err := w.ReadAll()
This will give you the metadata, the last raft.State and the slice of
raft.Entry items in the log.
*/
package wal

View file

@ -1,89 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"bufio"
"encoding/binary"
"hash"
"io"
"sync"
"github.com/coreos/etcd/pkg/crc"
"github.com/coreos/etcd/wal/walpb"
)
type encoder struct {
mu sync.Mutex
bw *bufio.Writer
crc hash.Hash32
buf []byte
uint64buf []byte
}
func newEncoder(w io.Writer, prevCrc uint32) *encoder {
return &encoder{
bw: bufio.NewWriter(w),
crc: crc.New(prevCrc, crcTable),
// 1MB buffer
buf: make([]byte, 1024*1024),
uint64buf: make([]byte, 8),
}
}
func (e *encoder) encode(rec *walpb.Record) error {
e.mu.Lock()
defer e.mu.Unlock()
e.crc.Write(rec.Data)
rec.Crc = e.crc.Sum32()
var (
data []byte
err error
n int
)
if rec.Size() > len(e.buf) {
data, err = rec.Marshal()
if err != nil {
return err
}
} else {
n, err = rec.MarshalTo(e.buf)
if err != nil {
return err
}
data = e.buf[:n]
}
if err = writeInt64(e.bw, int64(len(data)), e.uint64buf); err != nil {
return err
}
_, err = e.bw.Write(data)
return err
}
func (e *encoder) flush() error {
e.mu.Lock()
defer e.mu.Unlock()
return e.bw.Flush()
}
func writeInt64(w io.Writer, n int64, buf []byte) error {
// http://golang.org/src/encoding/binary/binary.go
binary.LittleEndian.PutUint64(buf, uint64(n))
_, err := w.Write(buf)
return err
}

View file

@ -1,38 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import "github.com/prometheus/client_golang/prometheus"
var (
syncDurations = prometheus.NewHistogram(prometheus.HistogramOpts{
Namespace: "etcd",
Subsystem: "wal",
Name: "fsync_durations_seconds",
Help: "The latency distributions of fsync called by wal.",
Buckets: prometheus.ExponentialBuckets(0.001, 2, 14),
})
lastIndexSaved = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "etcd",
Subsystem: "wal",
Name: "last_index_saved",
Help: "The index of the last entry saved by wal.",
})
)
func init() {
prometheus.MustRegister(syncDurations)
prometheus.MustRegister(lastIndexSaved)
}

View file

@ -1,45 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import "io"
type multiReadCloser struct {
closers []io.Closer
reader io.Reader
}
func (mc *multiReadCloser) Close() error {
var err error
for i := range mc.closers {
err = mc.closers[i].Close()
}
return err
}
func (mc *multiReadCloser) Read(p []byte) (int, error) {
return mc.reader.Read(p)
}
func MultiReadCloser(readClosers ...io.ReadCloser) io.ReadCloser {
cs := make([]io.Closer, len(readClosers))
rs := make([]io.Reader, len(readClosers))
for i := range readClosers {
cs[i] = readClosers[i]
rs[i] = readClosers[i]
}
r := io.MultiReader(rs...)
return &multiReadCloser{cs, r}
}

View file

@ -1,107 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"io"
"os"
"path"
"k8s.io/kubernetes/third_party/forked/etcd237/pkg/fileutil"
"github.com/coreos/etcd/wal/walpb"
)
// Repair tries to repair ErrUnexpectedEOF in the
// last wal file by truncating.
func Repair(dirpath string) bool {
f, err := openLast(dirpath)
if err != nil {
return false
}
defer f.Close()
n := 0
rec := &walpb.Record{}
decoder := newDecoder(f)
defer decoder.close()
for {
err := decoder.decode(rec)
switch err {
case nil:
n += 8 + rec.Size()
// update crc of the decoder when necessary
switch rec.Type {
case crcType:
crc := decoder.crc.Sum32()
// current crc of decoder must match the crc of the record.
// do no need to match 0 crc, since the decoder is a new one at this case.
if crc != 0 && rec.Validate(crc) != nil {
return false
}
decoder.updateCRC(rec.Crc)
}
continue
case io.EOF:
return true
case io.ErrUnexpectedEOF:
plog.Noticef("repairing %v", f.Name())
bf, bferr := os.Create(f.Name() + ".broken")
if bferr != nil {
plog.Errorf("could not repair %v, failed to create backup file", f.Name())
return false
}
defer bf.Close()
if _, err = f.Seek(0, os.SEEK_SET); err != nil {
plog.Errorf("could not repair %v, failed to read file", f.Name())
return false
}
if _, err = io.Copy(bf, f); err != nil {
plog.Errorf("could not repair %v, failed to copy file", f.Name())
return false
}
if err = f.Truncate(int64(n)); err != nil {
plog.Errorf("could not repair %v, failed to truncate file", f.Name())
return false
}
if err = f.Sync(); err != nil {
plog.Errorf("could not repair %v, failed to sync file", f.Name())
return false
}
return true
default:
plog.Errorf("could not repair error (%v)", err)
return false
}
}
}
// openLast opens the last wal file for read and write.
func openLast(dirpath string) (*os.File, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
last := path.Join(dirpath, names[len(names)-1])
return os.OpenFile(last, os.O_RDWR, 0)
}

View file

@ -1,93 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"errors"
"fmt"
"strings"
"k8s.io/kubernetes/third_party/forked/etcd237/pkg/fileutil"
)
var (
badWalName = errors.New("bad wal name")
)
func Exist(dirpath string) bool {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return false
}
return len(names) != 0
}
// searchIndex returns the last array index of names whose raft index section is
// equal to or smaller than the given index.
// The given names MUST be sorted.
func searchIndex(names []string, index uint64) (int, bool) {
for i := len(names) - 1; i >= 0; i-- {
name := names[i]
_, curIndex, err := parseWalName(name)
if err != nil {
plog.Panicf("parse correct name should never fail: %v", err)
}
if index >= curIndex {
return i, true
}
}
return -1, false
}
// names should have been sorted based on sequence number.
// isValidSeq checks whether seq increases continuously.
func isValidSeq(names []string) bool {
var lastSeq uint64
for _, name := range names {
curSeq, _, err := parseWalName(name)
if err != nil {
plog.Panicf("parse correct name should never fail: %v", err)
}
if lastSeq != 0 && lastSeq != curSeq-1 {
return false
}
lastSeq = curSeq
}
return true
}
func checkWalNames(names []string) []string {
wnames := make([]string, 0)
for _, name := range names {
if _, _, err := parseWalName(name); err != nil {
plog.Warningf("ignored file %v in wal", name)
continue
}
wnames = append(wnames, name)
}
return wnames
}
func parseWalName(str string) (seq, index uint64, err error) {
if !strings.HasSuffix(str, ".wal") {
return 0, 0, badWalName
}
_, err = fmt.Sscanf(str, "%016x-%016x.wal", &seq, &index)
return seq, index, err
}
func walName(seq, index uint64) string {
return fmt.Sprintf("%016x-%016x.wal", seq, index)
}

View file

@ -1,571 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package wal
import (
"errors"
"fmt"
"hash/crc32"
"io"
"os"
"path"
"reflect"
"sync"
"time"
"k8s.io/kubernetes/third_party/forked/etcd237/pkg/fileutil"
"github.com/coreos/etcd/pkg/pbutil"
"github.com/coreos/etcd/raft/raftpb"
"github.com/coreos/etcd/wal/walpb"
"github.com/coreos/pkg/capnslog"
)
const (
metadataType int64 = iota + 1
entryType
stateType
crcType
snapshotType
// the owner can make/remove files inside the directory
privateDirMode = 0700
// the expected size of each wal segment file.
// the actual size might be bigger than it.
segmentSizeBytes = 64 * 1000 * 1000 // 64MB
)
var (
plog = capnslog.NewPackageLogger("github.com/coreos/etcd", "wal")
ErrMetadataConflict = errors.New("wal: conflicting metadata found")
ErrFileNotFound = errors.New("wal: file not found")
ErrCRCMismatch = errors.New("wal: crc mismatch")
ErrSnapshotMismatch = errors.New("wal: snapshot mismatch")
ErrSnapshotNotFound = errors.New("wal: snapshot not found")
crcTable = crc32.MakeTable(crc32.Castagnoli)
)
// WAL is a logical representation of the stable storage.
// WAL is either in read mode or append mode but not both.
// A newly created WAL is in append mode, and ready for appending records.
// A just opened WAL is in read mode, and ready for reading records.
// The WAL will be ready for appending after reading out all the previous records.
type WAL struct {
dir string // the living directory of the underlay files
metadata []byte // metadata recorded at the head of each WAL
state raftpb.HardState // hardstate recorded at the head of WAL
start walpb.Snapshot // snapshot to start reading
decoder *decoder // decoder to decode records
mu sync.Mutex
f *os.File // underlay file opened for appending, sync
seq uint64 // sequence of the wal file currently used for writes
enti uint64 // index of the last entry saved to the wal
encoder *encoder // encoder to encode records
locks []fileutil.Lock // the file locks the WAL is holding (the name is increasing)
}
// Create creates a WAL ready for appending records. The given metadata is
// recorded at the head of each WAL file, and can be retrieved with ReadAll.
func Create(dirpath string, metadata []byte) (*WAL, error) {
if Exist(dirpath) {
return nil, os.ErrExist
}
if err := os.MkdirAll(dirpath, privateDirMode); err != nil {
return nil, err
}
p := path.Join(dirpath, walName(0, 0))
f, err := os.OpenFile(p, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0600)
if err != nil {
return nil, err
}
l, err := fileutil.NewLock(f.Name())
if err != nil {
return nil, err
}
if err = l.Lock(); err != nil {
return nil, err
}
w := &WAL{
dir: dirpath,
metadata: metadata,
seq: 0,
f: f,
encoder: newEncoder(f, 0),
}
w.locks = append(w.locks, l)
if err := w.saveCrc(0); err != nil {
return nil, err
}
if err := w.encoder.encode(&walpb.Record{Type: metadataType, Data: metadata}); err != nil {
return nil, err
}
if err := w.SaveSnapshot(walpb.Snapshot{}); err != nil {
return nil, err
}
return w, nil
}
// Open opens the WAL at the given snap.
// The snap SHOULD have been previously saved to the WAL, or the following
// ReadAll will fail.
// The returned WAL is ready to read and the first record will be the one after
// the given snap. The WAL cannot be appended to before reading out all of its
// previous records.
func Open(dirpath string, snap walpb.Snapshot) (*WAL, error) {
return openAtIndex(dirpath, snap, true)
}
// OpenForRead only opens the wal files for read.
// Write on a read only wal panics.
func OpenForRead(dirpath string, snap walpb.Snapshot) (*WAL, error) {
return openAtIndex(dirpath, snap, false)
}
func openAtIndex(dirpath string, snap walpb.Snapshot, write bool) (*WAL, error) {
names, err := fileutil.ReadDir(dirpath)
if err != nil {
return nil, err
}
names = checkWalNames(names)
if len(names) == 0 {
return nil, ErrFileNotFound
}
nameIndex, ok := searchIndex(names, snap.Index)
if !ok || !isValidSeq(names[nameIndex:]) {
return nil, ErrFileNotFound
}
// open the wal files for reading
rcs := make([]io.ReadCloser, 0)
ls := make([]fileutil.Lock, 0)
for _, name := range names[nameIndex:] {
f, err := os.Open(path.Join(dirpath, name))
if err != nil {
return nil, err
}
l, err := fileutil.NewLock(f.Name())
if err != nil {
return nil, err
}
err = l.TryLock()
if err != nil {
if write {
return nil, err
}
}
rcs = append(rcs, f)
ls = append(ls, l)
}
rc := MultiReadCloser(rcs...)
// create a WAL ready for reading
w := &WAL{
dir: dirpath,
start: snap,
decoder: newDecoder(rc),
locks: ls,
}
if write {
// open the last wal file for appending
seq, _, err := parseWalName(names[len(names)-1])
if err != nil {
rc.Close()
return nil, err
}
last := path.Join(dirpath, names[len(names)-1])
f, err := os.OpenFile(last, os.O_WRONLY|os.O_APPEND, 0)
if err != nil {
rc.Close()
return nil, err
}
err = fileutil.Preallocate(f, segmentSizeBytes)
if err != nil {
rc.Close()
plog.Errorf("failed to allocate space when creating new wal file (%v)", err)
return nil, err
}
w.f = f
w.seq = seq
}
return w, nil
}
// ReadAll reads out records of the current WAL.
// If opened in write mode, it must read out all records until EOF. Or an error
// will be returned.
// If opened in read mode, it will try to read all records if possible.
// If it cannot read out the expected snap, it will return ErrSnapshotNotFound.
// If loaded snap doesn't match with the expected one, it will return
// all the records and error ErrSnapshotMismatch.
// TODO: detect not-last-snap error.
// TODO: maybe loose the checking of match.
// After ReadAll, the WAL will be ready for appending new records.
func (w *WAL) ReadAll() (metadata []byte, state raftpb.HardState, ents []raftpb.Entry, err error) {
w.mu.Lock()
defer w.mu.Unlock()
rec := &walpb.Record{}
decoder := w.decoder
var match bool
for err = decoder.decode(rec); err == nil; err = decoder.decode(rec) {
switch rec.Type {
case entryType:
e := mustUnmarshalEntry(rec.Data)
if e.Index > w.start.Index {
ents = append(ents[:e.Index-w.start.Index-1], e)
}
w.enti = e.Index
case stateType:
state = mustUnmarshalState(rec.Data)
case metadataType:
if metadata != nil && !reflect.DeepEqual(metadata, rec.Data) {
state.Reset()
return nil, state, nil, ErrMetadataConflict
}
metadata = rec.Data
case crcType:
crc := decoder.crc.Sum32()
// current crc of decoder must match the crc of the record.
// do no need to match 0 crc, since the decoder is a new one at this case.
if crc != 0 && rec.Validate(crc) != nil {
state.Reset()
return nil, state, nil, ErrCRCMismatch
}
decoder.updateCRC(rec.Crc)
case snapshotType:
var snap walpb.Snapshot
pbutil.MustUnmarshal(&snap, rec.Data)
if snap.Index == w.start.Index {
if snap.Term != w.start.Term {
state.Reset()
return nil, state, nil, ErrSnapshotMismatch
}
match = true
}
default:
state.Reset()
return nil, state, nil, fmt.Errorf("unexpected block type %d", rec.Type)
}
}
switch w.f {
case nil:
// We do not have to read out all entries in read mode.
// The last record maybe a partial written one, so
// ErrunexpectedEOF might be returned.
if err != io.EOF && err != io.ErrUnexpectedEOF {
state.Reset()
return nil, state, nil, err
}
default:
// We must read all of the entries if WAL is opened in write mode.
if err != io.EOF {
state.Reset()
return nil, state, nil, err
}
}
err = nil
if !match {
err = ErrSnapshotNotFound
}
// close decoder, disable reading
w.decoder.close()
w.start = walpb.Snapshot{}
w.metadata = metadata
if w.f != nil {
// create encoder (chain crc with the decoder), enable appending
w.encoder = newEncoder(w.f, w.decoder.lastCRC())
w.decoder = nil
lastIndexSaved.Set(float64(w.enti))
}
return metadata, state, ents, err
}
// cut closes current file written and creates a new one ready to append.
// cut first creates a temp wal file and writes necessary headers into it.
// Then cut atomically rename temp wal file to a wal file.
func (w *WAL) cut() error {
// close old wal file
if err := w.sync(); err != nil {
return err
}
if err := w.f.Close(); err != nil {
return err
}
fpath := path.Join(w.dir, walName(w.seq+1, w.enti+1))
ftpath := fpath + ".tmp"
// create a temp wal file with name sequence + 1, or truncate the existing one
ft, err := os.OpenFile(ftpath, os.O_WRONLY|os.O_APPEND|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return err
}
// update writer and save the previous crc
w.f = ft
prevCrc := w.encoder.crc.Sum32()
w.encoder = newEncoder(w.f, prevCrc)
if err = w.saveCrc(prevCrc); err != nil {
return err
}
if err = w.encoder.encode(&walpb.Record{Type: metadataType, Data: w.metadata}); err != nil {
return err
}
if err = w.saveState(&w.state); err != nil {
return err
}
// close temp wal file
if err = w.sync(); err != nil {
return err
}
if err = w.f.Close(); err != nil {
return err
}
// atomically move temp wal file to wal file
if err = os.Rename(ftpath, fpath); err != nil {
return err
}
// open the wal file and update writer again
f, err := os.OpenFile(fpath, os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
return err
}
if err = fileutil.Preallocate(f, segmentSizeBytes); err != nil {
plog.Errorf("failed to allocate space when creating new wal file (%v)", err)
return err
}
w.f = f
prevCrc = w.encoder.crc.Sum32()
w.encoder = newEncoder(w.f, prevCrc)
// lock the new wal file
l, err := fileutil.NewLock(f.Name())
if err != nil {
return err
}
if err := l.Lock(); err != nil {
return err
}
w.locks = append(w.locks, l)
// increase the wal seq
w.seq++
plog.Infof("segmented wal file %v is created", fpath)
return nil
}
func (w *WAL) sync() error {
if w.encoder != nil {
if err := w.encoder.flush(); err != nil {
return err
}
}
start := time.Now()
err := fileutil.Fdatasync(w.f)
syncDurations.Observe(float64(time.Since(start)) / float64(time.Second))
return err
}
// ReleaseLockTo releases the locks, which has smaller index than the given index
// except the largest one among them.
// For example, if WAL is holding lock 1,2,3,4,5,6, ReleaseLockTo(4) will release
// lock 1,2 but keep 3. ReleaseLockTo(5) will release 1,2,3 but keep 4.
func (w *WAL) ReleaseLockTo(index uint64) error {
w.mu.Lock()
defer w.mu.Unlock()
var smaller int
found := false
for i, l := range w.locks {
_, lockIndex, err := parseWalName(path.Base(l.Name()))
if err != nil {
return err
}
if lockIndex >= index {
smaller = i - 1
found = true
break
}
}
// if no lock index is greater than the release index, we can
// release lock up to the last one(excluding).
if !found && len(w.locks) != 0 {
smaller = len(w.locks) - 1
}
if smaller <= 0 {
return nil
}
for i := 0; i < smaller; i++ {
w.locks[i].Unlock()
w.locks[i].Destroy()
}
w.locks = w.locks[smaller:]
return nil
}
func (w *WAL) Close() error {
w.mu.Lock()
defer w.mu.Unlock()
if w.f != nil {
if err := w.sync(); err != nil {
return err
}
if err := w.f.Close(); err != nil {
return err
}
}
for _, l := range w.locks {
err := l.Unlock()
if err != nil {
plog.Errorf("failed to unlock during closing wal: %s", err)
}
err = l.Destroy()
if err != nil {
plog.Errorf("failed to destroy lock during closing wal: %s", err)
}
}
return nil
}
func (w *WAL) saveEntry(e *raftpb.Entry) error {
// TODO: add MustMarshalTo to reduce one allocation.
b := pbutil.MustMarshal(e)
rec := &walpb.Record{Type: entryType, Data: b}
if err := w.encoder.encode(rec); err != nil {
return err
}
w.enti = e.Index
lastIndexSaved.Set(float64(w.enti))
return nil
}
func (w *WAL) saveState(s *raftpb.HardState) error {
if isEmptyHardState(*s) {
return nil
}
w.state = *s
b := pbutil.MustMarshal(s)
rec := &walpb.Record{Type: stateType, Data: b}
return w.encoder.encode(rec)
}
func (w *WAL) Save(st raftpb.HardState, ents []raftpb.Entry) error {
w.mu.Lock()
defer w.mu.Unlock()
// short cut, do not call sync
if isEmptyHardState(st) && len(ents) == 0 {
return nil
}
mustSync := mustSync(st, w.state, len(ents))
// TODO(xiangli): no more reference operator
for i := range ents {
if err := w.saveEntry(&ents[i]); err != nil {
return err
}
}
if err := w.saveState(&st); err != nil {
return err
}
fstat, err := w.f.Stat()
if err != nil {
return err
}
if fstat.Size() < segmentSizeBytes {
if mustSync {
return w.sync()
}
return nil
}
// TODO: add a test for this code path when refactoring the tests
return w.cut()
}
func (w *WAL) SaveSnapshot(e walpb.Snapshot) error {
w.mu.Lock()
defer w.mu.Unlock()
b := pbutil.MustMarshal(&e)
rec := &walpb.Record{Type: snapshotType, Data: b}
if err := w.encoder.encode(rec); err != nil {
return err
}
// update enti only when snapshot is ahead of last index
if w.enti < e.Index {
w.enti = e.Index
}
lastIndexSaved.Set(float64(w.enti))
return w.sync()
}
func (w *WAL) saveCrc(prevCrc uint32) error {
return w.encoder.encode(&walpb.Record{Type: crcType, Crc: prevCrc})
}
func mustSync(st, prevst raftpb.HardState, entsnum int) bool {
// Persistent state on all servers:
// (Updated on stable storage before responding to RPCs)
// currentTerm
// votedFor
// log entries[]
if entsnum != 0 || st.Vote != prevst.Vote || st.Term != prevst.Term {
return true
}
return false
}
func isHardStateEqual(a, b raftpb.HardState) bool {
return a.Term == b.Term && a.Vote == b.Vote && a.Commit == b.Commit
}
var emptyState = raftpb.HardState{}
func isEmptyHardState(st raftpb.HardState) bool {
return isHardStateEqual(st, emptyState)
}

View file

@ -1,36 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"record.go",
"record.pb.go",
],
deps = ["//vendor/github.com/golang/protobuf/proto:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)
filegroup(
name = "go_default_library_protos",
srcs = ["record.proto"],
visibility = ["//visibility:public"],
)

View file

@ -1,29 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package walpb
import "errors"
var (
ErrCRCMismatch = errors.New("walpb: crc mismatch")
)
func (rec *Record) Validate(crc uint32) error {
if rec.Crc == crc {
return nil
}
rec.Reset()
return ErrCRCMismatch
}

View file

@ -1,495 +0,0 @@
// Code generated by protoc-gen-gogo.
// source: record.proto
// DO NOT EDIT!
/*
Package walpb is a generated protocol buffer package.
It is generated from these files:
record.proto
It has these top-level messages:
Record
Snapshot
*/
package walpb
import (
"fmt"
proto "github.com/golang/protobuf/proto"
)
import math "math"
import io "io"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
type Record struct {
Type int64 `protobuf:"varint,1,opt,name=type" json:"type"`
Crc uint32 `protobuf:"varint,2,opt,name=crc" json:"crc"`
Data []byte `protobuf:"bytes,3,opt,name=data" json:"data,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Record) Reset() { *m = Record{} }
func (m *Record) String() string { return proto.CompactTextString(m) }
func (*Record) ProtoMessage() {}
type Snapshot struct {
Index uint64 `protobuf:"varint,1,opt,name=index" json:"index"`
Term uint64 `protobuf:"varint,2,opt,name=term" json:"term"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Snapshot) Reset() { *m = Snapshot{} }
func (m *Snapshot) String() string { return proto.CompactTextString(m) }
func (*Snapshot) ProtoMessage() {}
func init() {
proto.RegisterType((*Record)(nil), "walpb.Record")
proto.RegisterType((*Snapshot)(nil), "walpb.Snapshot")
}
func (m *Record) Marshal() (data []byte, err error) {
size := m.Size()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *Record) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
data[i] = 0x8
i++
i = encodeVarintRecord(data, i, uint64(m.Type))
data[i] = 0x10
i++
i = encodeVarintRecord(data, i, uint64(m.Crc))
if m.Data != nil {
data[i] = 0x1a
i++
i = encodeVarintRecord(data, i, uint64(len(m.Data)))
i += copy(data[i:], m.Data)
}
if m.XXX_unrecognized != nil {
i += copy(data[i:], m.XXX_unrecognized)
}
return i, nil
}
func (m *Snapshot) Marshal() (data []byte, err error) {
size := m.Size()
data = make([]byte, size)
n, err := m.MarshalTo(data)
if err != nil {
return nil, err
}
return data[:n], nil
}
func (m *Snapshot) MarshalTo(data []byte) (int, error) {
var i int
_ = i
var l int
_ = l
data[i] = 0x8
i++
i = encodeVarintRecord(data, i, uint64(m.Index))
data[i] = 0x10
i++
i = encodeVarintRecord(data, i, uint64(m.Term))
if m.XXX_unrecognized != nil {
i += copy(data[i:], m.XXX_unrecognized)
}
return i, nil
}
func encodeFixed64Record(data []byte, offset int, v uint64) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
data[offset+4] = uint8(v >> 32)
data[offset+5] = uint8(v >> 40)
data[offset+6] = uint8(v >> 48)
data[offset+7] = uint8(v >> 56)
return offset + 8
}
func encodeFixed32Record(data []byte, offset int, v uint32) int {
data[offset] = uint8(v)
data[offset+1] = uint8(v >> 8)
data[offset+2] = uint8(v >> 16)
data[offset+3] = uint8(v >> 24)
return offset + 4
}
func encodeVarintRecord(data []byte, offset int, v uint64) int {
for v >= 1<<7 {
data[offset] = uint8(v&0x7f | 0x80)
v >>= 7
offset++
}
data[offset] = uint8(v)
return offset + 1
}
func (m *Record) Size() (n int) {
var l int
_ = l
n += 1 + sovRecord(uint64(m.Type))
n += 1 + sovRecord(uint64(m.Crc))
if m.Data != nil {
l = len(m.Data)
n += 1 + l + sovRecord(uint64(l))
}
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func (m *Snapshot) Size() (n int) {
var l int
_ = l
n += 1 + sovRecord(uint64(m.Index))
n += 1 + sovRecord(uint64(m.Term))
if m.XXX_unrecognized != nil {
n += len(m.XXX_unrecognized)
}
return n
}
func sovRecord(x uint64) (n int) {
for {
n++
x >>= 7
if x == 0 {
break
}
}
return n
}
func sozRecord(x uint64) (n int) {
return sovRecord(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *Record) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Record: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Record: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType)
}
m.Type = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Type |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Crc", wireType)
}
m.Crc = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Crc |= (uint32(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType)
}
var byteLen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
byteLen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if byteLen < 0 {
return ErrInvalidLengthRecord
}
postIndex := iNdEx + byteLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Data = append(m.Data[:0], data[iNdEx:postIndex]...)
if m.Data == nil {
m.Data = []byte{}
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipRecord(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthRecord
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Snapshot) Unmarshal(data []byte) error {
l := len(data)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Snapshot: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Snapshot: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
m.Index = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Index |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Term", wireType)
}
m.Term = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRecord
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
m.Term |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipRecord(data[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthRecord
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
m.XXX_unrecognized = append(m.XXX_unrecognized, data[iNdEx:iNdEx+skippy]...)
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func skipRecord(data []byte) (n int, err error) {
l := len(data)
iNdEx := 0
for iNdEx < l {
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowRecord
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
wireType := int(wire & 0x7)
switch wireType {
case 0:
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowRecord
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
iNdEx++
if data[iNdEx-1] < 0x80 {
break
}
}
return iNdEx, nil
case 1:
iNdEx += 8
return iNdEx, nil
case 2:
var length int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowRecord
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
length |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
iNdEx += length
if length < 0 {
return 0, ErrInvalidLengthRecord
}
return iNdEx, nil
case 3:
for {
var innerWire uint64
var start int = iNdEx
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return 0, ErrIntOverflowRecord
}
if iNdEx >= l {
return 0, io.ErrUnexpectedEOF
}
b := data[iNdEx]
iNdEx++
innerWire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
innerWireType := int(innerWire & 0x7)
if innerWireType == 4 {
break
}
next, err := skipRecord(data[start:])
if err != nil {
return 0, err
}
iNdEx = start + next
}
return iNdEx, nil
case 4:
return iNdEx, nil
case 5:
iNdEx += 4
return iNdEx, nil
default:
return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
}
}
panic("unreachable")
}
var (
ErrInvalidLengthRecord = fmt.Errorf("proto: negative length found during unmarshaling")
ErrIntOverflowRecord = fmt.Errorf("proto: integer overflow")
)

View file

@ -1,20 +0,0 @@
syntax = "proto2";
package walpb;
import "gogoproto/gogo.proto";
option (gogoproto.marshaler_all) = true;
option (gogoproto.sizer_all) = true;
option (gogoproto.unmarshaler_all) = true;
option (gogoproto.goproto_getters_all) = false;
message Record {
optional int64 type = 1 [(gogoproto.nullable) = false];
optional uint32 crc = 2 [(gogoproto.nullable) = false];
optional bytes data = 3;
}
message Snapshot {
optional uint64 index = 1 [(gogoproto.nullable) = false];
optional uint64 term = 2 [(gogoproto.nullable) = false];
}

View file

@ -1,36 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_library(
name = "go_default_library",
srcs = [
"deep_equal.go",
"type.go",
],
)
go_test(
name = "go_default_test",
srcs = ["deep_equal_test.go"],
library = ":go_default_library",
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View file

@ -1,388 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package reflect is a fork of go's standard library reflection package, which
// allows for deep equal with equality functions defined.
package reflect
import (
"fmt"
"reflect"
"strings"
)
// Equalities is a map from type to a function comparing two values of
// that type.
type Equalities map[reflect.Type]reflect.Value
// For convenience, panics on errrors
func EqualitiesOrDie(funcs ...interface{}) Equalities {
e := Equalities{}
if err := e.AddFuncs(funcs...); err != nil {
panic(err)
}
return e
}
// AddFuncs is a shortcut for multiple calls to AddFunc.
func (e Equalities) AddFuncs(funcs ...interface{}) error {
for _, f := range funcs {
if err := e.AddFunc(f); err != nil {
return err
}
}
return nil
}
// AddFunc uses func as an equality function: it must take
// two parameters of the same type, and return a boolean.
func (e Equalities) AddFunc(eqFunc interface{}) error {
fv := reflect.ValueOf(eqFunc)
ft := fv.Type()
if ft.Kind() != reflect.Func {
return fmt.Errorf("expected func, got: %v", ft)
}
if ft.NumIn() != 2 {
return fmt.Errorf("expected three 'in' params, got: %v", ft)
}
if ft.NumOut() != 1 {
return fmt.Errorf("expected one 'out' param, got: %v", ft)
}
if ft.In(0) != ft.In(1) {
return fmt.Errorf("expected arg 1 and 2 to have same type, but got %v", ft)
}
var forReturnType bool
boolType := reflect.TypeOf(forReturnType)
if ft.Out(0) != boolType {
return fmt.Errorf("expected bool return, got: %v", ft)
}
e[ft.In(0)] = fv
return nil
}
// Below here is forked from go's reflect/deepequal.go
// During deepValueEqual, must keep track of checks that are
// in progress. The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited comparisons are stored in a map indexed by visit.
type visit struct {
a1 uintptr
a2 uintptr
typ reflect.Type
}
// unexportedTypePanic is thrown when you use this DeepEqual on something that has an
// unexported type. It indicates a programmer error, so should not occur at runtime,
// which is why it's not public and thus impossible to catch.
type unexportedTypePanic []reflect.Type
func (u unexportedTypePanic) Error() string { return u.String() }
func (u unexportedTypePanic) String() string {
strs := make([]string, len(u))
for i, t := range u {
strs[i] = fmt.Sprintf("%v", t)
}
return "an unexported field was encountered, nested like this: " + strings.Join(strs, " -> ")
}
func makeUsefulPanic(v reflect.Value) {
if x := recover(); x != nil {
if u, ok := x.(unexportedTypePanic); ok {
u = append(unexportedTypePanic{v.Type()}, u...)
x = u
}
panic(x)
}
}
// Tests for deep equality using reflected types. The map argument tracks
// comparisons that have already been seen, which allows short circuiting on
// recursive types.
func (e Equalities) deepValueEqual(v1, v2 reflect.Value, visited map[visit]bool, depth int) bool {
defer makeUsefulPanic(v1)
if !v1.IsValid() || !v2.IsValid() {
return v1.IsValid() == v2.IsValid()
}
if v1.Type() != v2.Type() {
return false
}
if fv, ok := e[v1.Type()]; ok {
return fv.Call([]reflect.Value{v1, v2})[0].Bool()
}
hard := func(k reflect.Kind) bool {
switch k {
case reflect.Array, reflect.Map, reflect.Slice, reflect.Struct:
return true
}
return false
}
if v1.CanAddr() && v2.CanAddr() && hard(v1.Kind()) {
addr1 := v1.UnsafeAddr()
addr2 := v2.UnsafeAddr()
if addr1 > addr2 {
// Canonicalize order to reduce number of entries in visited.
addr1, addr2 = addr2, addr1
}
// Short circuit if references are identical ...
if addr1 == addr2 {
return true
}
// ... or already seen
typ := v1.Type()
v := visit{addr1, addr2, typ}
if visited[v] {
return true
}
// Remember for later.
visited[v] = true
}
switch v1.Kind() {
case reflect.Array:
// We don't need to check length here because length is part of
// an array's type, which has already been filtered for.
for i := 0; i < v1.Len(); i++ {
if !e.deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
return false
}
}
return true
case reflect.Slice:
if (v1.IsNil() || v1.Len() == 0) != (v2.IsNil() || v2.Len() == 0) {
return false
}
if v1.IsNil() || v1.Len() == 0 {
return true
}
if v1.Len() != v2.Len() {
return false
}
if v1.Pointer() == v2.Pointer() {
return true
}
for i := 0; i < v1.Len(); i++ {
if !e.deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
return false
}
}
return true
case reflect.Interface:
if v1.IsNil() || v2.IsNil() {
return v1.IsNil() == v2.IsNil()
}
return e.deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
case reflect.Ptr:
return e.deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
case reflect.Struct:
for i, n := 0, v1.NumField(); i < n; i++ {
if !e.deepValueEqual(v1.Field(i), v2.Field(i), visited, depth+1) {
return false
}
}
return true
case reflect.Map:
if (v1.IsNil() || v1.Len() == 0) != (v2.IsNil() || v2.Len() == 0) {
return false
}
if v1.IsNil() || v1.Len() == 0 {
return true
}
if v1.Len() != v2.Len() {
return false
}
if v1.Pointer() == v2.Pointer() {
return true
}
for _, k := range v1.MapKeys() {
if !e.deepValueEqual(v1.MapIndex(k), v2.MapIndex(k), visited, depth+1) {
return false
}
}
return true
case reflect.Func:
if v1.IsNil() && v2.IsNil() {
return true
}
// Can't do better than this:
return false
default:
// Normal equality suffices
if !v1.CanInterface() || !v2.CanInterface() {
panic(unexportedTypePanic{})
}
return v1.Interface() == v2.Interface()
}
}
// DeepEqual is like reflect.DeepEqual, but focused on semantic equality
// instead of memory equality.
//
// It will use e's equality functions if it finds types that match.
//
// An empty slice *is* equal to a nil slice for our purposes; same for maps.
//
// Unexported field members cannot be compared and will cause an imformative panic; you must add an Equality
// function for these types.
func (e Equalities) DeepEqual(a1, a2 interface{}) bool {
if a1 == nil || a2 == nil {
return a1 == a2
}
v1 := reflect.ValueOf(a1)
v2 := reflect.ValueOf(a2)
if v1.Type() != v2.Type() {
return false
}
return e.deepValueEqual(v1, v2, make(map[visit]bool), 0)
}
func (e Equalities) deepValueDerive(v1, v2 reflect.Value, visited map[visit]bool, depth int) bool {
defer makeUsefulPanic(v1)
if !v1.IsValid() || !v2.IsValid() {
return v1.IsValid() == v2.IsValid()
}
if v1.Type() != v2.Type() {
return false
}
if fv, ok := e[v1.Type()]; ok {
return fv.Call([]reflect.Value{v1, v2})[0].Bool()
}
hard := func(k reflect.Kind) bool {
switch k {
case reflect.Array, reflect.Map, reflect.Slice, reflect.Struct:
return true
}
return false
}
if v1.CanAddr() && v2.CanAddr() && hard(v1.Kind()) {
addr1 := v1.UnsafeAddr()
addr2 := v2.UnsafeAddr()
if addr1 > addr2 {
// Canonicalize order to reduce number of entries in visited.
addr1, addr2 = addr2, addr1
}
// Short circuit if references are identical ...
if addr1 == addr2 {
return true
}
// ... or already seen
typ := v1.Type()
v := visit{addr1, addr2, typ}
if visited[v] {
return true
}
// Remember for later.
visited[v] = true
}
switch v1.Kind() {
case reflect.Array:
// We don't need to check length here because length is part of
// an array's type, which has already been filtered for.
for i := 0; i < v1.Len(); i++ {
if !e.deepValueDerive(v1.Index(i), v2.Index(i), visited, depth+1) {
return false
}
}
return true
case reflect.Slice:
if v1.IsNil() || v1.Len() == 0 {
return true
}
if v1.Len() > v2.Len() {
return false
}
if v1.Pointer() == v2.Pointer() {
return true
}
for i := 0; i < v1.Len(); i++ {
if !e.deepValueDerive(v1.Index(i), v2.Index(i), visited, depth+1) {
return false
}
}
return true
case reflect.String:
if v1.Len() == 0 {
return true
}
if v1.Len() > v2.Len() {
return false
}
return v1.String() == v2.String()
case reflect.Interface:
if v1.IsNil() {
return true
}
return e.deepValueDerive(v1.Elem(), v2.Elem(), visited, depth+1)
case reflect.Ptr:
if v1.IsNil() {
return true
}
return e.deepValueDerive(v1.Elem(), v2.Elem(), visited, depth+1)
case reflect.Struct:
for i, n := 0, v1.NumField(); i < n; i++ {
if !e.deepValueDerive(v1.Field(i), v2.Field(i), visited, depth+1) {
return false
}
}
return true
case reflect.Map:
if v1.IsNil() || v1.Len() == 0 {
return true
}
if v1.Len() > v2.Len() {
return false
}
if v1.Pointer() == v2.Pointer() {
return true
}
for _, k := range v1.MapKeys() {
if !e.deepValueDerive(v1.MapIndex(k), v2.MapIndex(k), visited, depth+1) {
return false
}
}
return true
case reflect.Func:
if v1.IsNil() && v2.IsNil() {
return true
}
// Can't do better than this:
return false
default:
// Normal equality suffices
if !v1.CanInterface() || !v2.CanInterface() {
panic(unexportedTypePanic{})
}
return v1.Interface() == v2.Interface()
}
}
// DeepDerivative is similar to DeepEqual except that unset fields in a1 are
// ignored (not compared). This allows us to focus on the fields that matter to
// the semantic comparison.
//
// The unset fields include a nil pointer and an empty string.
func (e Equalities) DeepDerivative(a1, a2 interface{}) bool {
if a1 == nil {
return true
}
v1 := reflect.ValueOf(a1)
v2 := reflect.ValueOf(a2)
if v1.Type() != v2.Type() {
return false
}
return e.deepValueDerive(v1, v2, make(map[visit]bool), 0)
}

View file

@ -1,137 +0,0 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package reflect
import (
"testing"
)
func TestEqualities(t *testing.T) {
e := Equalities{}
type Bar struct {
X int
}
type Baz struct {
Y Bar
}
err := e.AddFuncs(
func(a, b int) bool {
return a+1 == b
},
func(a, b Bar) bool {
return a.X*10 == b.X
},
)
if err != nil {
t.Fatalf("Unexpected: %v", err)
}
type Foo struct {
X int
}
table := []struct {
a, b interface{}
equal bool
}{
{1, 2, true},
{2, 1, false},
{"foo", "fo", false},
{"foo", "foo", true},
{"foo", "foobar", false},
{Foo{1}, Foo{2}, true},
{Foo{2}, Foo{1}, false},
{Bar{1}, Bar{10}, true},
{&Bar{1}, &Bar{10}, true},
{Baz{Bar{1}}, Baz{Bar{10}}, true},
{[...]string{}, [...]string{"1", "2", "3"}, false},
{[...]string{"1"}, [...]string{"1", "2", "3"}, false},
{[...]string{"1", "2", "3"}, [...]string{}, false},
{[...]string{"1", "2", "3"}, [...]string{"1", "2", "3"}, true},
{map[string]int{"foo": 1}, map[string]int{}, false},
{map[string]int{"foo": 1}, map[string]int{"foo": 2}, true},
{map[string]int{"foo": 2}, map[string]int{"foo": 1}, false},
{map[string]int{"foo": 1}, map[string]int{"foo": 2, "bar": 6}, false},
{map[string]int{"foo": 1, "bar": 6}, map[string]int{"foo": 2}, false},
{map[string]int{}, map[string]int(nil), true},
{[]string(nil), []string(nil), true},
{[]string{}, []string(nil), true},
{[]string(nil), []string{}, true},
{[]string{"1"}, []string(nil), false},
{[]string{}, []string{"1", "2", "3"}, false},
{[]string{"1"}, []string{"1", "2", "3"}, false},
{[]string{"1", "2", "3"}, []string{}, false},
}
for _, item := range table {
if e, a := item.equal, e.DeepEqual(item.a, item.b); e != a {
t.Errorf("Expected (%+v == %+v) == %v, but got %v", item.a, item.b, e, a)
}
}
}
func TestDerivates(t *testing.T) {
e := Equalities{}
type Bar struct {
X int
}
type Baz struct {
Y Bar
}
err := e.AddFuncs(
func(a, b int) bool {
return a+1 == b
},
func(a, b Bar) bool {
return a.X*10 == b.X
},
)
if err != nil {
t.Fatalf("Unexpected: %v", err)
}
type Foo struct {
X int
}
table := []struct {
a, b interface{}
equal bool
}{
{1, 2, true},
{2, 1, false},
{"foo", "fo", false},
{"foo", "foo", true},
{"foo", "foobar", false},
{Foo{1}, Foo{2}, true},
{Foo{2}, Foo{1}, false},
{Bar{1}, Bar{10}, true},
{&Bar{1}, &Bar{10}, true},
{Baz{Bar{1}}, Baz{Bar{10}}, true},
{[...]string{}, [...]string{"1", "2", "3"}, false},
{[...]string{"1"}, [...]string{"1", "2", "3"}, false},
{[...]string{"1", "2", "3"}, [...]string{}, false},
{[...]string{"1", "2", "3"}, [...]string{"1", "2", "3"}, true},
{map[string]int{"foo": 1}, map[string]int{}, false},
{map[string]int{"foo": 1}, map[string]int{"foo": 2}, true},
{map[string]int{"foo": 2}, map[string]int{"foo": 1}, false},
{map[string]int{"foo": 1}, map[string]int{"foo": 2, "bar": 6}, true},
{map[string]int{"foo": 1, "bar": 6}, map[string]int{"foo": 2}, false},
{map[string]int{}, map[string]int(nil), true},
{[]string(nil), []string(nil), true},
{[]string{}, []string(nil), true},
{[]string(nil), []string{}, true},
{[]string{"1"}, []string(nil), false},
{[]string{}, []string{"1", "2", "3"}, true},
{[]string{"1"}, []string{"1", "2", "3"}, true},
{[]string{"1", "2", "3"}, []string{}, false},
}
for _, item := range table {
if e, a := item.equal, e.DeepDerivative(item.a, item.b); e != a {
t.Errorf("Expected (%+v ~ %+v) == %v, but got %v", item.a, item.b, e, a)
}
}
}

View file

@ -1,91 +0,0 @@
//This package is copied from Go library reflect/type.go.
//The struct tag library provides no way to extract the list of struct tags, only
//a specific tag
package reflect
import (
"fmt"
"strconv"
"strings"
)
type StructTag struct {
Name string
Value string
}
func (t StructTag) String() string {
return fmt.Sprintf("%s:%q", t.Name, t.Value)
}
type StructTags []StructTag
func (tags StructTags) String() string {
s := make([]string, 0, len(tags))
for _, tag := range tags {
s = append(s, tag.String())
}
return "`" + strings.Join(s, " ") + "`"
}
func (tags StructTags) Has(name string) bool {
for i := range tags {
if tags[i].Name == name {
return true
}
}
return false
}
// ParseStructTags returns the full set of fields in a struct tag in the order they appear in
// the struct tag.
func ParseStructTags(tag string) (StructTags, error) {
tags := StructTags{}
for tag != "" {
// Skip leading space.
i := 0
for i < len(tag) && tag[i] == ' ' {
i++
}
tag = tag[i:]
if tag == "" {
break
}
// Scan to colon. A space, a quote or a control character is a syntax error.
// Strictly speaking, control chars include the range [0x7f, 0x9f], not just
// [0x00, 0x1f], but in practice, we ignore the multi-byte control characters
// as it is simpler to inspect the tag's bytes than the tag's runes.
i = 0
for i < len(tag) && tag[i] > ' ' && tag[i] != ':' && tag[i] != '"' && tag[i] != 0x7f {
i++
}
if i == 0 || i+1 >= len(tag) || tag[i] != ':' || tag[i+1] != '"' {
break
}
name := string(tag[:i])
tag = tag[i+1:]
// Scan quoted string to find value.
i = 1
for i < len(tag) && tag[i] != '"' {
if tag[i] == '\\' {
i++
}
i++
}
if i >= len(tag) {
break
}
qvalue := string(tag[:i+1])
tag = tag[i+1:]
value, err := strconv.Unquote(qvalue)
if err != nil {
return nil, err
}
tags = append(tags, StructTag{Name: name, Value: value})
}
return tags, nil
}

View file

@ -1,29 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"exec.go",
"funcs.go",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View file

@ -1,94 +0,0 @@
//This package is copied from Go library text/template.
//The original private functions indirect and printableValue
//are exported as public functions.
package template
import (
"fmt"
"reflect"
)
var Indirect = indirect
var PrintableValue = printableValue
var (
errorType = reflect.TypeOf((*error)(nil)).Elem()
fmtStringerType = reflect.TypeOf((*fmt.Stringer)(nil)).Elem()
)
// indirect returns the item at the end of indirection, and a bool to indicate if it's nil.
// We indirect through pointers and empty interfaces (only) because
// non-empty interfaces have methods we might need.
func indirect(v reflect.Value) (rv reflect.Value, isNil bool) {
for ; v.Kind() == reflect.Ptr || v.Kind() == reflect.Interface; v = v.Elem() {
if v.IsNil() {
return v, true
}
if v.Kind() == reflect.Interface && v.NumMethod() > 0 {
break
}
}
return v, false
}
// printableValue returns the, possibly indirected, interface value inside v that
// is best for a call to formatted printer.
func printableValue(v reflect.Value) (interface{}, bool) {
if v.Kind() == reflect.Ptr {
v, _ = indirect(v) // fmt.Fprint handles nil.
}
if !v.IsValid() {
return "<no value>", true
}
if !v.Type().Implements(errorType) && !v.Type().Implements(fmtStringerType) {
if v.CanAddr() && (reflect.PtrTo(v.Type()).Implements(errorType) || reflect.PtrTo(v.Type()).Implements(fmtStringerType)) {
v = v.Addr()
} else {
switch v.Kind() {
case reflect.Chan, reflect.Func:
return nil, false
}
}
}
return v.Interface(), true
}
// canBeNil reports whether an untyped nil can be assigned to the type. See reflect.Zero.
func canBeNil(typ reflect.Type) bool {
switch typ.Kind() {
case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
return true
}
return false
}
// isTrue reports whether the value is 'true', in the sense of not the zero of its type,
// and whether the value has a meaningful truth value.
func isTrue(val reflect.Value) (truth, ok bool) {
if !val.IsValid() {
// Something like var x interface{}, never set. It's a form of nil.
return false, true
}
switch val.Kind() {
case reflect.Array, reflect.Map, reflect.Slice, reflect.String:
truth = val.Len() > 0
case reflect.Bool:
truth = val.Bool()
case reflect.Complex64, reflect.Complex128:
truth = val.Complex() != 0
case reflect.Chan, reflect.Func, reflect.Ptr, reflect.Interface:
truth = !val.IsNil()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
truth = val.Int() != 0
case reflect.Float32, reflect.Float64:
truth = val.Float() != 0
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
truth = val.Uint() != 0
case reflect.Struct:
truth = true // Struct values are always true.
default:
return
}
return truth, true
}

View file

@ -1,599 +0,0 @@
//This package is copied from Go library text/template.
//The original private functions eq, ge, gt, le, lt, and ne
//are exported as public functions.
package template
import (
"bytes"
"errors"
"fmt"
"io"
"net/url"
"reflect"
"strings"
"unicode"
"unicode/utf8"
)
var Equal = eq
var GreaterEqual = ge
var Greater = gt
var LessEqual = le
var Less = lt
var NotEqual = ne
// FuncMap is the type of the map defining the mapping from names to functions.
// Each function must have either a single return value, or two return values of
// which the second has type error. In that case, if the second (error)
// return value evaluates to non-nil during execution, execution terminates and
// Execute returns that error.
type FuncMap map[string]interface{}
var builtins = FuncMap{
"and": and,
"call": call,
"html": HTMLEscaper,
"index": index,
"js": JSEscaper,
"len": length,
"not": not,
"or": or,
"print": fmt.Sprint,
"printf": fmt.Sprintf,
"println": fmt.Sprintln,
"urlquery": URLQueryEscaper,
// Comparisons
"eq": eq, // ==
"ge": ge, // >=
"gt": gt, // >
"le": le, // <=
"lt": lt, // <
"ne": ne, // !=
}
var builtinFuncs = createValueFuncs(builtins)
// createValueFuncs turns a FuncMap into a map[string]reflect.Value
func createValueFuncs(funcMap FuncMap) map[string]reflect.Value {
m := make(map[string]reflect.Value)
addValueFuncs(m, funcMap)
return m
}
// addValueFuncs adds to values the functions in funcs, converting them to reflect.Values.
func addValueFuncs(out map[string]reflect.Value, in FuncMap) {
for name, fn := range in {
v := reflect.ValueOf(fn)
if v.Kind() != reflect.Func {
panic("value for " + name + " not a function")
}
if !goodFunc(v.Type()) {
panic(fmt.Errorf("can't install method/function %q with %d results", name, v.Type().NumOut()))
}
out[name] = v
}
}
// AddFuncs adds to values the functions in funcs. It does no checking of the input -
// call addValueFuncs first.
func addFuncs(out, in FuncMap) {
for name, fn := range in {
out[name] = fn
}
}
// goodFunc checks that the function or method has the right result signature.
func goodFunc(typ reflect.Type) bool {
// We allow functions with 1 result or 2 results where the second is an error.
switch {
case typ.NumOut() == 1:
return true
case typ.NumOut() == 2 && typ.Out(1) == errorType:
return true
}
return false
}
// findFunction looks for a function in the template, and global map.
func findFunction(name string) (reflect.Value, bool) {
if fn := builtinFuncs[name]; fn.IsValid() {
return fn, true
}
return reflect.Value{}, false
}
// Indexing.
// index returns the result of indexing its first argument by the following
// arguments. Thus "index x 1 2 3" is, in Go syntax, x[1][2][3]. Each
// indexed item must be a map, slice, or array.
func index(item interface{}, indices ...interface{}) (interface{}, error) {
v := reflect.ValueOf(item)
for _, i := range indices {
index := reflect.ValueOf(i)
var isNil bool
if v, isNil = indirect(v); isNil {
return nil, fmt.Errorf("index of nil pointer")
}
switch v.Kind() {
case reflect.Array, reflect.Slice, reflect.String:
var x int64
switch index.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
x = index.Int()
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
x = int64(index.Uint())
default:
return nil, fmt.Errorf("cannot index slice/array with type %s", index.Type())
}
if x < 0 || x >= int64(v.Len()) {
return nil, fmt.Errorf("index out of range: %d", x)
}
v = v.Index(int(x))
case reflect.Map:
if !index.IsValid() {
index = reflect.Zero(v.Type().Key())
}
if !index.Type().AssignableTo(v.Type().Key()) {
return nil, fmt.Errorf("%s is not index type for %s", index.Type(), v.Type())
}
if x := v.MapIndex(index); x.IsValid() {
v = x
} else {
v = reflect.Zero(v.Type().Elem())
}
default:
return nil, fmt.Errorf("can't index item of type %s", v.Type())
}
}
return v.Interface(), nil
}
// Length
// length returns the length of the item, with an error if it has no defined length.
func length(item interface{}) (int, error) {
v, isNil := indirect(reflect.ValueOf(item))
if isNil {
return 0, fmt.Errorf("len of nil pointer")
}
switch v.Kind() {
case reflect.Array, reflect.Chan, reflect.Map, reflect.Slice, reflect.String:
return v.Len(), nil
}
return 0, fmt.Errorf("len of type %s", v.Type())
}
// Function invocation
// call returns the result of evaluating the first argument as a function.
// The function must return 1 result, or 2 results, the second of which is an error.
func call(fn interface{}, args ...interface{}) (interface{}, error) {
v := reflect.ValueOf(fn)
typ := v.Type()
if typ.Kind() != reflect.Func {
return nil, fmt.Errorf("non-function of type %s", typ)
}
if !goodFunc(typ) {
return nil, fmt.Errorf("function called with %d args; should be 1 or 2", typ.NumOut())
}
numIn := typ.NumIn()
var dddType reflect.Type
if typ.IsVariadic() {
if len(args) < numIn-1 {
return nil, fmt.Errorf("wrong number of args: got %d want at least %d", len(args), numIn-1)
}
dddType = typ.In(numIn - 1).Elem()
} else {
if len(args) != numIn {
return nil, fmt.Errorf("wrong number of args: got %d want %d", len(args), numIn)
}
}
argv := make([]reflect.Value, len(args))
for i, arg := range args {
value := reflect.ValueOf(arg)
// Compute the expected type. Clumsy because of variadics.
var argType reflect.Type
if !typ.IsVariadic() || i < numIn-1 {
argType = typ.In(i)
} else {
argType = dddType
}
if !value.IsValid() && canBeNil(argType) {
value = reflect.Zero(argType)
}
if !value.Type().AssignableTo(argType) {
return nil, fmt.Errorf("arg %d has type %s; should be %s", i, value.Type(), argType)
}
argv[i] = value
}
result := v.Call(argv)
if len(result) == 2 && !result[1].IsNil() {
return result[0].Interface(), result[1].Interface().(error)
}
return result[0].Interface(), nil
}
// Boolean logic.
func truth(a interface{}) bool {
t, _ := isTrue(reflect.ValueOf(a))
return t
}
// and computes the Boolean AND of its arguments, returning
// the first false argument it encounters, or the last argument.
func and(arg0 interface{}, args ...interface{}) interface{} {
if !truth(arg0) {
return arg0
}
for i := range args {
arg0 = args[i]
if !truth(arg0) {
break
}
}
return arg0
}
// or computes the Boolean OR of its arguments, returning
// the first true argument it encounters, or the last argument.
func or(arg0 interface{}, args ...interface{}) interface{} {
if truth(arg0) {
return arg0
}
for i := range args {
arg0 = args[i]
if truth(arg0) {
break
}
}
return arg0
}
// not returns the Boolean negation of its argument.
func not(arg interface{}) (truth bool) {
truth, _ = isTrue(reflect.ValueOf(arg))
return !truth
}
// Comparison.
// TODO: Perhaps allow comparison between signed and unsigned integers.
var (
errBadComparisonType = errors.New("invalid type for comparison")
errBadComparison = errors.New("incompatible types for comparison")
errNoComparison = errors.New("missing argument for comparison")
)
type kind int
const (
invalidKind kind = iota
boolKind
complexKind
intKind
floatKind
integerKind
stringKind
uintKind
)
func basicKind(v reflect.Value) (kind, error) {
switch v.Kind() {
case reflect.Bool:
return boolKind, nil
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return intKind, nil
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
return uintKind, nil
case reflect.Float32, reflect.Float64:
return floatKind, nil
case reflect.Complex64, reflect.Complex128:
return complexKind, nil
case reflect.String:
return stringKind, nil
}
return invalidKind, errBadComparisonType
}
// eq evaluates the comparison a == b || a == c || ...
func eq(arg1 interface{}, arg2 ...interface{}) (bool, error) {
v1 := reflect.ValueOf(arg1)
k1, err := basicKind(v1)
if err != nil {
return false, err
}
if len(arg2) == 0 {
return false, errNoComparison
}
for _, arg := range arg2 {
v2 := reflect.ValueOf(arg)
k2, err := basicKind(v2)
if err != nil {
return false, err
}
truth := false
if k1 != k2 {
// Special case: Can compare integer values regardless of type's sign.
switch {
case k1 == intKind && k2 == uintKind:
truth = v1.Int() >= 0 && uint64(v1.Int()) == v2.Uint()
case k1 == uintKind && k2 == intKind:
truth = v2.Int() >= 0 && v1.Uint() == uint64(v2.Int())
default:
return false, errBadComparison
}
} else {
switch k1 {
case boolKind:
truth = v1.Bool() == v2.Bool()
case complexKind:
truth = v1.Complex() == v2.Complex()
case floatKind:
truth = v1.Float() == v2.Float()
case intKind:
truth = v1.Int() == v2.Int()
case stringKind:
truth = v1.String() == v2.String()
case uintKind:
truth = v1.Uint() == v2.Uint()
default:
panic("invalid kind")
}
}
if truth {
return true, nil
}
}
return false, nil
}
// ne evaluates the comparison a != b.
func ne(arg1, arg2 interface{}) (bool, error) {
// != is the inverse of ==.
equal, err := eq(arg1, arg2)
return !equal, err
}
// lt evaluates the comparison a < b.
func lt(arg1, arg2 interface{}) (bool, error) {
v1 := reflect.ValueOf(arg1)
k1, err := basicKind(v1)
if err != nil {
return false, err
}
v2 := reflect.ValueOf(arg2)
k2, err := basicKind(v2)
if err != nil {
return false, err
}
truth := false
if k1 != k2 {
// Special case: Can compare integer values regardless of type's sign.
switch {
case k1 == intKind && k2 == uintKind:
truth = v1.Int() < 0 || uint64(v1.Int()) < v2.Uint()
case k1 == uintKind && k2 == intKind:
truth = v2.Int() >= 0 && v1.Uint() < uint64(v2.Int())
default:
return false, errBadComparison
}
} else {
switch k1 {
case boolKind, complexKind:
return false, errBadComparisonType
case floatKind:
truth = v1.Float() < v2.Float()
case intKind:
truth = v1.Int() < v2.Int()
case stringKind:
truth = v1.String() < v2.String()
case uintKind:
truth = v1.Uint() < v2.Uint()
default:
panic("invalid kind")
}
}
return truth, nil
}
// le evaluates the comparison <= b.
func le(arg1, arg2 interface{}) (bool, error) {
// <= is < or ==.
lessThan, err := lt(arg1, arg2)
if lessThan || err != nil {
return lessThan, err
}
return eq(arg1, arg2)
}
// gt evaluates the comparison a > b.
func gt(arg1, arg2 interface{}) (bool, error) {
// > is the inverse of <=.
lessOrEqual, err := le(arg1, arg2)
if err != nil {
return false, err
}
return !lessOrEqual, nil
}
// ge evaluates the comparison a >= b.
func ge(arg1, arg2 interface{}) (bool, error) {
// >= is the inverse of <.
lessThan, err := lt(arg1, arg2)
if err != nil {
return false, err
}
return !lessThan, nil
}
// HTML escaping.
var (
htmlQuot = []byte("&#34;") // shorter than "&quot;"
htmlApos = []byte("&#39;") // shorter than "&apos;" and apos was not in HTML until HTML5
htmlAmp = []byte("&amp;")
htmlLt = []byte("&lt;")
htmlGt = []byte("&gt;")
)
// HTMLEscape writes to w the escaped HTML equivalent of the plain text data b.
func HTMLEscape(w io.Writer, b []byte) {
last := 0
for i, c := range b {
var html []byte
switch c {
case '"':
html = htmlQuot
case '\'':
html = htmlApos
case '&':
html = htmlAmp
case '<':
html = htmlLt
case '>':
html = htmlGt
default:
continue
}
w.Write(b[last:i])
w.Write(html)
last = i + 1
}
w.Write(b[last:])
}
// HTMLEscapeString returns the escaped HTML equivalent of the plain text data s.
func HTMLEscapeString(s string) string {
// Avoid allocation if we can.
if strings.IndexAny(s, `'"&<>`) < 0 {
return s
}
var b bytes.Buffer
HTMLEscape(&b, []byte(s))
return b.String()
}
// HTMLEscaper returns the escaped HTML equivalent of the textual
// representation of its arguments.
func HTMLEscaper(args ...interface{}) string {
return HTMLEscapeString(evalArgs(args))
}
// JavaScript escaping.
var (
jsLowUni = []byte(`\u00`)
hex = []byte("0123456789ABCDEF")
jsBackslash = []byte(`\\`)
jsApos = []byte(`\'`)
jsQuot = []byte(`\"`)
jsLt = []byte(`\x3C`)
jsGt = []byte(`\x3E`)
)
// JSEscape writes to w the escaped JavaScript equivalent of the plain text data b.
func JSEscape(w io.Writer, b []byte) {
last := 0
for i := 0; i < len(b); i++ {
c := b[i]
if !jsIsSpecial(rune(c)) {
// fast path: nothing to do
continue
}
w.Write(b[last:i])
if c < utf8.RuneSelf {
// Quotes, slashes and angle brackets get quoted.
// Control characters get written as \u00XX.
switch c {
case '\\':
w.Write(jsBackslash)
case '\'':
w.Write(jsApos)
case '"':
w.Write(jsQuot)
case '<':
w.Write(jsLt)
case '>':
w.Write(jsGt)
default:
w.Write(jsLowUni)
t, b := c>>4, c&0x0f
w.Write(hex[t : t+1])
w.Write(hex[b : b+1])
}
} else {
// Unicode rune.
r, size := utf8.DecodeRune(b[i:])
if unicode.IsPrint(r) {
w.Write(b[i : i+size])
} else {
fmt.Fprintf(w, "\\u%04X", r)
}
i += size - 1
}
last = i + 1
}
w.Write(b[last:])
}
// JSEscapeString returns the escaped JavaScript equivalent of the plain text data s.
func JSEscapeString(s string) string {
// Avoid allocation if we can.
if strings.IndexFunc(s, jsIsSpecial) < 0 {
return s
}
var b bytes.Buffer
JSEscape(&b, []byte(s))
return b.String()
}
func jsIsSpecial(r rune) bool {
switch r {
case '\\', '\'', '"', '<', '>':
return true
}
return r < ' ' || utf8.RuneSelf <= r
}
// JSEscaper returns the escaped JavaScript equivalent of the textual
// representation of its arguments.
func JSEscaper(args ...interface{}) string {
return JSEscapeString(evalArgs(args))
}
// URLQueryEscaper returns the escaped value of the textual representation of
// its arguments in a form suitable for embedding in a URL query.
func URLQueryEscaper(args ...interface{}) string {
return url.QueryEscape(evalArgs(args))
}
// evalArgs formats the list of arguments into a string. It is therefore equivalent to
// fmt.Sprint(args...)
// except that each argument is indirected (if a pointer), as required,
// using the same rules as the default string evaluation during template
// execution.
func evalArgs(args []interface{}) string {
ok := false
var s string
// Fast path for simple common case.
if len(args) == 1 {
s, ok = args[0].(string)
}
if !ok {
for i, arg := range args {
a, ok := printableValue(reflect.ValueOf(arg))
if ok {
args[i] = a
} // else left fmt do its thing
}
s = fmt.Sprint(args...)
}
return s
}

View file

@ -1,31 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["graph.go"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//third_party/forked/gonum/graph/internal/linear:all-srcs",
"//third_party/forked/gonum/graph/simple:all-srcs",
"//third_party/forked/gonum/graph/traverse:all-srcs",
],
tags = ["automanaged"],
)

View file

@ -1,23 +0,0 @@
Copyright ©2013 The gonum Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the gonum project nor the names of its authors and
contributors may be used to endorse or promote products derived from this
software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View file

@ -1 +0,0 @@
Forked from gonum/graph@50b27dea7ebbfb052dfaf91681afc6fde28d8796 to support memory-use improvements to the simple graph

View file

@ -1,153 +0,0 @@
// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package graph
// Node is a graph node. It returns a graph-unique integer ID.
type Node interface {
ID() int
}
// Edge is a graph edge. In directed graphs, the direction of the
// edge is given from -> to, otherwise the edge is semantically
// unordered.
type Edge interface {
From() Node
To() Node
Weight() float64
}
// Graph is a generalized graph.
type Graph interface {
// Has returns whether the node exists within the graph.
Has(Node) bool
// Nodes returns all the nodes in the graph.
Nodes() []Node
// From returns all nodes that can be reached directly
// from the given node.
From(Node) []Node
// HasEdgeBeteen returns whether an edge exists between
// nodes x and y without considering direction.
HasEdgeBetween(x, y Node) bool
// Edge returns the edge from u to v if such an edge
// exists and nil otherwise. The node v must be directly
// reachable from u as defined by the From method.
Edge(u, v Node) Edge
}
// Undirected is an undirected graph.
type Undirected interface {
Graph
// EdgeBetween returns the edge between nodes x and y.
EdgeBetween(x, y Node) Edge
}
// Directed is a directed graph.
type Directed interface {
Graph
// HasEdgeFromTo returns whether an edge exists
// in the graph from u to v.
HasEdgeFromTo(u, v Node) bool
// To returns all nodes that can reach directly
// to the given node.
To(Node) []Node
}
// Weighter defines graphs that can report edge weights.
type Weighter interface {
// Weight returns the weight for the edge between
// x and y if Edge(x, y) returns a non-nil Edge.
// If x and y are the same node or there is no
// joining edge between the two nodes the weight
// value returned is implementation dependent.
// Weight returns true if an edge exists between
// x and y or if x and y have the same ID, false
// otherwise.
Weight(x, y Node) (w float64, ok bool)
}
// NodeAdder is an interface for adding arbitrary nodes to a graph.
type NodeAdder interface {
// NewNodeID returns a new unique arbitrary ID.
NewNodeID() int
// Adds a node to the graph. AddNode panics if
// the added node ID matches an existing node ID.
AddNode(Node)
}
// NodeRemover is an interface for removing nodes from a graph.
type NodeRemover interface {
// RemoveNode removes a node from the graph, as
// well as any edges attached to it. If the node
// is not in the graph it is a no-op.
RemoveNode(Node)
}
// EdgeSetter is an interface for adding edges to a graph.
type EdgeSetter interface {
// SetEdge adds an edge from one node to another.
// If the graph supports node addition the nodes
// will be added if they do not exist, otherwise
// SetEdge will panic.
// If the IDs returned by e.From and e.To are
// equal, SetEdge will panic.
SetEdge(e Edge)
}
// EdgeRemover is an interface for removing nodes from a graph.
type EdgeRemover interface {
// RemoveEdge removes the given edge, leaving the
// terminal nodes. If the edge does not exist it
// is a no-op.
RemoveEdge(Edge)
}
// Builder is a graph that can have nodes and edges added.
type Builder interface {
NodeAdder
EdgeSetter
}
// UndirectedBuilder is an undirected graph builder.
type UndirectedBuilder interface {
Undirected
Builder
}
// DirectedBuilder is a directed graph builder.
type DirectedBuilder interface {
Directed
Builder
}
// Copy copies nodes and edges as undirected edges from the source to the destination
// without first clearing the destination. Copy will panic if a node ID in the source
// graph matches a node ID in the destination.
//
// If the source is undirected and the destination is directed both directions will
// be present in the destination after the copy is complete.
//
// If the source is a directed graph, the destination is undirected, and a fundamental
// cycle exists with two nodes where the edge weights differ, the resulting destination
// graph's edge weight between those nodes is undefined. If there is a defined function
// to resolve such conflicts, an Undirect may be used to do this.
func Copy(dst Builder, src Graph) {
nodes := src.Nodes()
for _, n := range nodes {
dst.AddNode(n)
}
for _, u := range nodes {
for _, v := range src.From(u) {
dst.SetEdge(src.Edge(u, v))
}
}
}

View file

@ -1,27 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["linear.go"],
deps = ["//third_party/forked/gonum/graph:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View file

@ -1,74 +0,0 @@
// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package linear provides common linear data structures.
package linear
import (
"k8s.io/kubernetes/third_party/forked/gonum/graph"
)
// NodeStack implements a LIFO stack of graph.Node.
type NodeStack []graph.Node
// Len returns the number of graph.Nodes on the stack.
func (s *NodeStack) Len() int { return len(*s) }
// Pop returns the last graph.Node on the stack and removes it
// from the stack.
func (s *NodeStack) Pop() graph.Node {
v := *s
v, n := v[:len(v)-1], v[len(v)-1]
*s = v
return n
}
// Push adds the node n to the stack at the last position.
func (s *NodeStack) Push(n graph.Node) { *s = append(*s, n) }
// NodeQueue implements a FIFO queue.
type NodeQueue struct {
head int
data []graph.Node
}
// Len returns the number of graph.Nodes in the queue.
func (q *NodeQueue) Len() int { return len(q.data) - q.head }
// Enqueue adds the node n to the back of the queue.
func (q *NodeQueue) Enqueue(n graph.Node) {
if len(q.data) == cap(q.data) && q.head > 0 {
l := q.Len()
copy(q.data, q.data[q.head:])
q.head = 0
q.data = append(q.data[:l], n)
} else {
q.data = append(q.data, n)
}
}
// Dequeue returns the graph.Node at the front of the queue and
// removes it from the queue.
func (q *NodeQueue) Dequeue() graph.Node {
if q.Len() == 0 {
panic("queue: empty queue")
}
var n graph.Node
n, q.data[q.head] = q.data[q.head], nil
q.head++
if q.Len() == 0 {
q.head = 0
q.data = q.data[:0]
}
return n
}
// Reset clears the queue for reuse.
func (q *NodeQueue) Reset() {
q.head = 0
q.data = q.data[:0]
}

View file

@ -1,47 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
go_test(
name = "go_default_test",
srcs = [
"directed_acyclic_test.go",
"edgeholder_test.go",
"undirected_test.go",
],
library = ":go_default_library",
deps = ["//third_party/forked/gonum/graph:go_default_library"],
)
go_library(
name = "go_default_library",
srcs = [
"directed_acyclic.go",
"edgeholder.go",
"simple.go",
"undirected.go",
],
deps = [
"//third_party/forked/gonum/graph:go_default_library",
"//vendor/golang.org/x/tools/container/intsets:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View file

@ -1,83 +0,0 @@
package simple
import (
"k8s.io/kubernetes/third_party/forked/gonum/graph"
)
// DirectedAcyclicGraph implements graph.Directed using UndirectedGraph,
// which only stores one edge for any node pair.
type DirectedAcyclicGraph struct {
*UndirectedGraph
}
func NewDirectedAcyclicGraph(self, absent float64) *DirectedAcyclicGraph {
return &DirectedAcyclicGraph{
UndirectedGraph: NewUndirectedGraph(self, absent),
}
}
func (g *DirectedAcyclicGraph) HasEdgeFromTo(u, v graph.Node) bool {
edge := g.UndirectedGraph.EdgeBetween(u, v)
if edge == nil {
return false
}
return (edge.From().ID() == u.ID())
}
func (g *DirectedAcyclicGraph) From(n graph.Node) []graph.Node {
if !g.Has(n) {
return nil
}
fid := n.ID()
nodes := make([]graph.Node, 0, g.UndirectedGraph.edges[n.ID()].Len())
g.UndirectedGraph.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) {
if edge.From().ID() == fid {
nodes = append(nodes, g.UndirectedGraph.nodes[edge.To().ID()])
}
})
return nodes
}
func (g *DirectedAcyclicGraph) VisitFrom(n graph.Node, visitor func(neighbor graph.Node) (shouldContinue bool)) {
if !g.Has(n) {
return
}
fid := n.ID()
g.UndirectedGraph.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) {
if edge.From().ID() == fid {
if !visitor(g.UndirectedGraph.nodes[edge.To().ID()]) {
return
}
}
})
}
func (g *DirectedAcyclicGraph) To(n graph.Node) []graph.Node {
if !g.Has(n) {
return nil
}
tid := n.ID()
nodes := make([]graph.Node, 0, g.UndirectedGraph.edges[n.ID()].Len())
g.UndirectedGraph.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) {
if edge.To().ID() == tid {
nodes = append(nodes, g.UndirectedGraph.nodes[edge.From().ID()])
}
})
return nodes
}
func (g *DirectedAcyclicGraph) VisitTo(n graph.Node, visitor func(neighbor graph.Node) (shouldContinue bool)) {
if !g.Has(n) {
return
}
tid := n.ID()
g.UndirectedGraph.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) {
if edge.To().ID() == tid {
if !visitor(g.UndirectedGraph.nodes[edge.From().ID()]) {
return
}
}
})
}

View file

@ -1,62 +0,0 @@
// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package simple
import (
"math"
"testing"
"k8s.io/kubernetes/third_party/forked/gonum/graph"
)
var _ graph.Graph = &DirectedAcyclicGraph{}
var _ graph.Directed = &DirectedAcyclicGraph{}
// Tests Issue #27
func TestAcyclicEdgeOvercounting(t *testing.T) {
g := generateDummyAcyclicGraph()
if neigh := g.From(Node(Node(2))); len(neigh) != 2 {
t.Errorf("Node 2 has incorrect number of neighbors got neighbors %v (count %d), expected 2 neighbors {0,1}", neigh, len(neigh))
}
}
func generateDummyAcyclicGraph() *DirectedAcyclicGraph {
nodes := [4]struct{ srcID, targetID int }{
{2, 1},
{1, 0},
{0, 2},
{2, 0},
}
g := NewDirectedAcyclicGraph(0, math.Inf(1))
for _, n := range nodes {
g.SetEdge(Edge{F: Node(n.srcID), T: Node(n.targetID), W: 1})
}
return g
}
// Test for issue #123 https://github.com/gonum/graph/issues/123
func TestAcyclicIssue123DirectedGraph(t *testing.T) {
defer func() {
if r := recover(); r != nil {
t.Errorf("unexpected panic: %v", r)
}
}()
g := NewDirectedAcyclicGraph(0, math.Inf(1))
n0 := Node(g.NewNodeID())
g.AddNode(n0)
n1 := Node(g.NewNodeID())
g.AddNode(n1)
g.RemoveNode(n0)
n2 := Node(g.NewNodeID())
g.AddNode(n2)
}

View file

@ -1,122 +0,0 @@
package simple
import "k8s.io/kubernetes/third_party/forked/gonum/graph"
// edgeHolder represents a set of edges, with no more than one edge to or from a particular neighbor node
type edgeHolder interface {
// Visit invokes visitor with each edge and the id of the neighbor node in the edge
Visit(visitor func(neighbor int, edge graph.Edge))
// Delete removes edges to or from the specified neighbor
Delete(neighbor int) edgeHolder
// Set stores the edge to or from the specified neighbor
Set(neighbor int, edge graph.Edge) edgeHolder
// Get returns the edge to or from the specified neighbor
Get(neighbor int) (graph.Edge, bool)
// Len returns the number of edges
Len() int
}
// sliceEdgeHolder holds a list of edges to or from self
type sliceEdgeHolder struct {
self int
edges []graph.Edge
}
func (e *sliceEdgeHolder) Visit(visitor func(neighbor int, edge graph.Edge)) {
for _, edge := range e.edges {
if edge.From().ID() == e.self {
visitor(edge.To().ID(), edge)
} else {
visitor(edge.From().ID(), edge)
}
}
}
func (e *sliceEdgeHolder) Delete(neighbor int) edgeHolder {
edges := e.edges[:0]
for i, edge := range e.edges {
if edge.From().ID() == e.self {
if edge.To().ID() == neighbor {
continue
}
} else {
if edge.From().ID() == neighbor {
continue
}
}
edges = append(edges, e.edges[i])
}
e.edges = edges
return e
}
func (e *sliceEdgeHolder) Set(neighbor int, newEdge graph.Edge) edgeHolder {
for i, edge := range e.edges {
if edge.From().ID() == e.self {
if edge.To().ID() == neighbor {
e.edges[i] = newEdge
return e
}
} else {
if edge.From().ID() == neighbor {
e.edges[i] = newEdge
return e
}
}
}
if len(e.edges) < 4 {
e.edges = append(e.edges, newEdge)
return e
}
h := mapEdgeHolder(make(map[int]graph.Edge, len(e.edges)+1))
for i, edge := range e.edges {
if edge.From().ID() == e.self {
h[edge.To().ID()] = e.edges[i]
} else {
h[edge.From().ID()] = e.edges[i]
}
}
h[neighbor] = newEdge
return h
}
func (e *sliceEdgeHolder) Get(neighbor int) (graph.Edge, bool) {
for _, edge := range e.edges {
if edge.From().ID() == e.self {
if edge.To().ID() == neighbor {
return edge, true
}
} else {
if edge.From().ID() == neighbor {
return edge, true
}
}
}
return nil, false
}
func (e *sliceEdgeHolder) Len() int {
return len(e.edges)
}
// mapEdgeHolder holds a map of neighbors to edges
type mapEdgeHolder map[int]graph.Edge
func (e mapEdgeHolder) Visit(visitor func(neighbor int, edge graph.Edge)) {
for neighbor, edge := range e {
visitor(neighbor, edge)
}
}
func (e mapEdgeHolder) Delete(neighbor int) edgeHolder {
delete(e, neighbor)
return e
}
func (e mapEdgeHolder) Set(neighbor int, edge graph.Edge) edgeHolder {
e[neighbor] = edge
return e
}
func (e mapEdgeHolder) Get(neighbor int) (graph.Edge, bool) {
edge, ok := e[neighbor]
return edge, ok
}
func (e mapEdgeHolder) Len() int {
return len(e)
}

View file

@ -1,104 +0,0 @@
package simple
import (
"reflect"
"sort"
"testing"
"k8s.io/kubernetes/third_party/forked/gonum/graph"
)
func TestEdgeHolder(t *testing.T) {
holder := edgeHolder(&sliceEdgeHolder{self: 1})
// Empty tests
if len := holder.Len(); len != 0 {
t.Errorf("expected 0")
}
if n, ok := holder.Get(2); ok || n != nil {
t.Errorf("expected nil,false")
}
holder.Visit(func(_ int, _ graph.Edge) { t.Errorf("unexpected call to visitor") })
holder = holder.Delete(2)
// Insert an edge to ourselves
holder = holder.Set(1, Edge{F: Node(1), T: Node(1)})
if len := holder.Len(); len != 1 {
t.Errorf("expected 1")
}
if n, ok := holder.Get(1); !ok || n == nil || n.From().ID() != 1 || n.To().ID() != 1 {
t.Errorf("expected edge to ourselves, got %#v", n)
}
neighbors := []int{}
holder.Visit(func(neighbor int, _ graph.Edge) { neighbors = append(neighbors, neighbor) })
if !reflect.DeepEqual(neighbors, []int{1}) {
t.Errorf("expected a single visit to ourselves, got %v", neighbors)
}
// Insert edges from us to other nodes
holder = holder.Set(2, Edge{F: Node(1), T: Node(2)})
holder = holder.Set(3, Edge{F: Node(1), T: Node(3)})
holder = holder.Set(4, Edge{F: Node(1), T: Node(4)})
if len := holder.Len(); len != 4 {
t.Errorf("expected 4")
}
if n, ok := holder.Get(2); !ok || n == nil || n.From().ID() != 1 || n.To().ID() != 2 {
t.Errorf("expected edge from us to another node, got %#v", n)
}
neighbors = []int{}
holder.Visit(func(neighbor int, _ graph.Edge) { neighbors = append(neighbors, neighbor) })
if !reflect.DeepEqual(neighbors, []int{1, 2, 3, 4}) {
t.Errorf("expected a single visit to ourselves, got %v", neighbors)
}
// Insert edges to us to other nodes
holder = holder.Set(2, Edge{F: Node(2), T: Node(1)})
holder = holder.Set(3, Edge{F: Node(3), T: Node(1)})
holder = holder.Set(4, Edge{F: Node(4), T: Node(1)})
if len := holder.Len(); len != 4 {
t.Errorf("expected 4")
}
if n, ok := holder.Get(2); !ok || n == nil || n.From().ID() != 2 || n.To().ID() != 1 {
t.Errorf("expected reversed edge, got %#v", n)
}
neighbors = []int{}
holder.Visit(func(neighbor int, _ graph.Edge) { neighbors = append(neighbors, neighbor) })
if !reflect.DeepEqual(neighbors, []int{1, 2, 3, 4}) {
t.Errorf("expected a single visit to ourselves, got %v", neighbors)
}
if _, ok := holder.(*sliceEdgeHolder); !ok {
t.Errorf("expected slice edge holder")
}
// Make the transition to a map
holder = holder.Set(5, Edge{F: Node(5), T: Node(1)})
if _, ok := holder.(mapEdgeHolder); !ok {
t.Errorf("expected map edge holder")
}
if len := holder.Len(); len != 5 {
t.Errorf("expected 5")
}
if n, ok := holder.Get(2); !ok || n == nil || n.From().ID() != 2 || n.To().ID() != 1 {
t.Errorf("expected old edges, got %#v", n)
}
if n, ok := holder.Get(5); !ok || n == nil || n.From().ID() != 5 || n.To().ID() != 1 {
t.Errorf("expected new edge, got %#v", n)
}
neighbors = []int{}
holder.Visit(func(neighbor int, _ graph.Edge) { neighbors = append(neighbors, neighbor) })
sort.Ints(neighbors) // sort, map order is random
if !reflect.DeepEqual(neighbors, []int{1, 2, 3, 4, 5}) {
t.Errorf("expected 1,2,3,4,5, got %v", neighbors)
}
holder = holder.Delete(1)
holder = holder.Delete(2)
holder = holder.Delete(3)
holder = holder.Delete(4)
holder = holder.Delete(5)
holder = holder.Delete(6)
if len := holder.Len(); len != 0 {
t.Errorf("expected 0")
}
}

View file

@ -1,45 +0,0 @@
// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package simple provides a suite of simple graph implementations satisfying
// the gonum/graph interfaces.
package simple
import (
"math"
"k8s.io/kubernetes/third_party/forked/gonum/graph"
)
// Node is a simple graph node.
type Node int
// ID returns the ID number of the node.
func (n Node) ID() int {
return int(n)
}
// Edge is a simple graph edge.
type Edge struct {
F, T graph.Node
W float64
}
// From returns the from-node of the edge.
func (e Edge) From() graph.Node { return e.F }
// To returns the to-node of the edge.
func (e Edge) To() graph.Node { return e.T }
// Weight returns the weight of the edge.
func (e Edge) Weight() float64 { return e.W }
// maxInt is the maximum value of the machine-dependent int type.
const maxInt int = int(^uint(0) >> 1)
// isSame returns whether two float64 values are the same where NaN values
// are equalable.
func isSame(a, b float64) bool {
return a == b || (math.IsNaN(a) && math.IsNaN(b))
}

View file

@ -1,242 +0,0 @@
// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package simple
import (
"fmt"
"golang.org/x/tools/container/intsets"
"k8s.io/kubernetes/third_party/forked/gonum/graph"
)
// UndirectedGraph implements a generalized undirected graph.
type UndirectedGraph struct {
nodes map[int]graph.Node
edges map[int]edgeHolder
self, absent float64
freeIDs intsets.Sparse
usedIDs intsets.Sparse
}
// NewUndirectedGraph returns an UndirectedGraph with the specified self and absent
// edge weight values.
func NewUndirectedGraph(self, absent float64) *UndirectedGraph {
return &UndirectedGraph{
nodes: make(map[int]graph.Node),
edges: make(map[int]edgeHolder),
self: self,
absent: absent,
}
}
// NewNodeID returns a new unique ID for a node to be added to g. The returned ID does
// not become a valid ID in g until it is added to g.
func (g *UndirectedGraph) NewNodeID() int {
if len(g.nodes) == 0 {
return 0
}
if len(g.nodes) == maxInt {
panic(fmt.Sprintf("simple: cannot allocate node: no slot"))
}
var id int
if g.freeIDs.Len() != 0 && g.freeIDs.TakeMin(&id) {
return id
}
if id = g.usedIDs.Max(); id < maxInt {
return id + 1
}
for id = 0; id < maxInt; id++ {
if !g.usedIDs.Has(id) {
return id
}
}
panic("unreachable")
}
// AddNode adds n to the graph. It panics if the added node ID matches an existing node ID.
func (g *UndirectedGraph) AddNode(n graph.Node) {
if _, exists := g.nodes[n.ID()]; exists {
panic(fmt.Sprintf("simple: node ID collision: %d", n.ID()))
}
g.nodes[n.ID()] = n
g.edges[n.ID()] = &sliceEdgeHolder{self: n.ID()}
g.freeIDs.Remove(n.ID())
g.usedIDs.Insert(n.ID())
}
// RemoveNode removes n from the graph, as well as any edges attached to it. If the node
// is not in the graph it is a no-op.
func (g *UndirectedGraph) RemoveNode(n graph.Node) {
if _, ok := g.nodes[n.ID()]; !ok {
return
}
delete(g.nodes, n.ID())
g.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) {
g.edges[neighbor] = g.edges[neighbor].Delete(n.ID())
})
delete(g.edges, n.ID())
g.freeIDs.Insert(n.ID())
g.usedIDs.Remove(n.ID())
}
// SetEdge adds e, an edge from one node to another. If the nodes do not exist, they are added.
// It will panic if the IDs of the e.From and e.To are equal.
func (g *UndirectedGraph) SetEdge(e graph.Edge) {
var (
from = e.From()
fid = from.ID()
to = e.To()
tid = to.ID()
)
if fid == tid {
panic("simple: adding self edge")
}
if !g.Has(from) {
g.AddNode(from)
}
if !g.Has(to) {
g.AddNode(to)
}
g.edges[fid] = g.edges[fid].Set(tid, e)
g.edges[tid] = g.edges[tid].Set(fid, e)
}
// RemoveEdge removes e from the graph, leaving the terminal nodes. If the edge does not exist
// it is a no-op.
func (g *UndirectedGraph) RemoveEdge(e graph.Edge) {
from, to := e.From(), e.To()
if _, ok := g.nodes[from.ID()]; !ok {
return
}
if _, ok := g.nodes[to.ID()]; !ok {
return
}
g.edges[from.ID()] = g.edges[from.ID()].Delete(to.ID())
g.edges[to.ID()] = g.edges[to.ID()].Delete(from.ID())
}
// Node returns the node in the graph with the given ID.
func (g *UndirectedGraph) Node(id int) graph.Node {
return g.nodes[id]
}
// Has returns whether the node exists within the graph.
func (g *UndirectedGraph) Has(n graph.Node) bool {
_, ok := g.nodes[n.ID()]
return ok
}
// Nodes returns all the nodes in the graph.
func (g *UndirectedGraph) Nodes() []graph.Node {
nodes := make([]graph.Node, len(g.nodes))
i := 0
for _, n := range g.nodes {
nodes[i] = n
i++
}
return nodes
}
// Edges returns all the edges in the graph.
func (g *UndirectedGraph) Edges() []graph.Edge {
var edges []graph.Edge
seen := make(map[[2]int]struct{})
for _, u := range g.edges {
u.Visit(func(neighbor int, e graph.Edge) {
uid := e.From().ID()
vid := e.To().ID()
if _, ok := seen[[2]int{uid, vid}]; ok {
return
}
seen[[2]int{uid, vid}] = struct{}{}
seen[[2]int{vid, uid}] = struct{}{}
edges = append(edges, e)
})
}
return edges
}
// From returns all nodes in g that can be reached directly from n.
func (g *UndirectedGraph) From(n graph.Node) []graph.Node {
if !g.Has(n) {
return nil
}
nodes := make([]graph.Node, g.edges[n.ID()].Len())
i := 0
g.edges[n.ID()].Visit(func(neighbor int, edge graph.Edge) {
nodes[i] = g.nodes[neighbor]
i++
})
return nodes
}
// HasEdgeBetween returns whether an edge exists between nodes x and y.
func (g *UndirectedGraph) HasEdgeBetween(x, y graph.Node) bool {
_, ok := g.edges[x.ID()].Get(y.ID())
return ok
}
// Edge returns the edge from u to v if such an edge exists and nil otherwise.
// The node v must be directly reachable from u as defined by the From method.
func (g *UndirectedGraph) Edge(u, v graph.Node) graph.Edge {
return g.EdgeBetween(u, v)
}
// EdgeBetween returns the edge between nodes x and y.
func (g *UndirectedGraph) EdgeBetween(x, y graph.Node) graph.Edge {
// We don't need to check if neigh exists because
// it's implicit in the edges access.
if !g.Has(x) {
return nil
}
edge, _ := g.edges[x.ID()].Get(y.ID())
return edge
}
// Weight returns the weight for the edge between x and y if Edge(x, y) returns a non-nil Edge.
// If x and y are the same node or there is no joining edge between the two nodes the weight
// value returned is either the graph's absent or self value. Weight returns true if an edge
// exists between x and y or if x and y have the same ID, false otherwise.
func (g *UndirectedGraph) Weight(x, y graph.Node) (w float64, ok bool) {
xid := x.ID()
yid := y.ID()
if xid == yid {
return g.self, true
}
if n, ok := g.edges[xid]; ok {
if e, ok := n.Get(yid); ok {
return e.Weight(), true
}
}
return g.absent, false
}
// Degree returns the degree of n in g.
func (g *UndirectedGraph) Degree(n graph.Node) int {
if _, ok := g.nodes[n.ID()]; !ok {
return 0
}
return g.edges[n.ID()].Len()
}

View file

@ -1,63 +0,0 @@
// Copyright ©2014 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package simple
import (
"math"
"testing"
"k8s.io/kubernetes/third_party/forked/gonum/graph"
)
var _ graph.Graph = (*UndirectedGraph)(nil)
func TestAssertMutableNotDirected(t *testing.T) {
var g graph.UndirectedBuilder = NewUndirectedGraph(0, math.Inf(1))
if _, ok := g.(graph.Directed); ok {
t.Fatal("Graph is directed, but a MutableGraph cannot safely be directed!")
}
}
func TestMaxID(t *testing.T) {
g := NewUndirectedGraph(0, math.Inf(1))
nodes := make(map[graph.Node]struct{})
for i := Node(0); i < 3; i++ {
g.AddNode(i)
nodes[i] = struct{}{}
}
g.RemoveNode(Node(0))
delete(nodes, Node(0))
g.RemoveNode(Node(2))
delete(nodes, Node(2))
n := Node(g.NewNodeID())
g.AddNode(n)
if !g.Has(n) {
t.Error("added node does not exist in graph")
}
if _, exists := nodes[n]; exists {
t.Errorf("Created already existing node id: %v", n.ID())
}
}
// Test for issue #123 https://github.com/gonum/graph/issues/123
func TestIssue123UndirectedGraph(t *testing.T) {
defer func() {
if r := recover(); r != nil {
t.Errorf("unexpected panic: %v", r)
}
}()
g := NewUndirectedGraph(0, math.Inf(1))
n0 := Node(g.NewNodeID())
g.AddNode(n0)
n1 := Node(g.NewNodeID())
g.AddNode(n1)
g.RemoveNode(n0)
n2 := Node(g.NewNodeID())
g.AddNode(n2)
}

View file

@ -1,34 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"traverse.go",
"visit_depth_first.go",
],
deps = [
"//third_party/forked/gonum/graph:go_default_library",
"//third_party/forked/gonum/graph/internal/linear:go_default_library",
"//vendor/golang.org/x/tools/container/intsets:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View file

@ -1,186 +0,0 @@
// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package traverse provides basic graph traversal primitives.
package traverse
import (
"golang.org/x/tools/container/intsets"
"k8s.io/kubernetes/third_party/forked/gonum/graph"
"k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear"
)
// BreadthFirst implements stateful breadth-first graph traversal.
type BreadthFirst struct {
EdgeFilter func(graph.Edge) bool
Visit func(u, v graph.Node)
queue linear.NodeQueue
visited *intsets.Sparse
}
// Walk performs a breadth-first traversal of the graph g starting from the given node,
// depending on the the EdgeFilter field and the until parameter if they are non-nil. The
// traversal follows edges for which EdgeFilter(edge) is true and returns the first node
// for which until(node, depth) is true. During the traversal, if the Visit field is
// non-nil, it is called with the nodes joined by each followed edge.
func (b *BreadthFirst) Walk(g graph.Graph, from graph.Node, until func(n graph.Node, d int) bool) graph.Node {
if b.visited == nil {
b.visited = &intsets.Sparse{}
}
b.queue.Enqueue(from)
b.visited.Insert(from.ID())
var (
depth int
children int
untilNext = 1
)
for b.queue.Len() > 0 {
t := b.queue.Dequeue()
if until != nil && until(t, depth) {
return t
}
for _, n := range g.From(t) {
if b.EdgeFilter != nil && !b.EdgeFilter(g.Edge(t, n)) {
continue
}
if b.visited.Has(n.ID()) {
continue
}
if b.Visit != nil {
b.Visit(t, n)
}
b.visited.Insert(n.ID())
children++
b.queue.Enqueue(n)
}
if untilNext--; untilNext == 0 {
depth++
untilNext = children
children = 0
}
}
return nil
}
// WalkAll calls Walk for each unvisited node of the graph g using edges independent
// of their direction. The functions before and after are called prior to commencing
// and after completing each walk if they are non-nil respectively. The function
// during is called on each node as it is traversed.
func (b *BreadthFirst) WalkAll(g graph.Undirected, before, after func(), during func(graph.Node)) {
b.Reset()
for _, from := range g.Nodes() {
if b.Visited(from) {
continue
}
if before != nil {
before()
}
b.Walk(g, from, func(n graph.Node, _ int) bool {
if during != nil {
during(n)
}
return false
})
if after != nil {
after()
}
}
}
// Visited returned whether the node n was visited during a traverse.
func (b *BreadthFirst) Visited(n graph.Node) bool {
return b.visited != nil && b.visited.Has(n.ID())
}
// Reset resets the state of the traverser for reuse.
func (b *BreadthFirst) Reset() {
b.queue.Reset()
if b.visited != nil {
b.visited.Clear()
}
}
// DepthFirst implements stateful depth-first graph traversal.
type DepthFirst struct {
EdgeFilter func(graph.Edge) bool
Visit func(u, v graph.Node)
stack linear.NodeStack
visited *intsets.Sparse
}
// Walk performs a depth-first traversal of the graph g starting from the given node,
// depending on the the EdgeFilter field and the until parameter if they are non-nil. The
// traversal follows edges for which EdgeFilter(edge) is true and returns the first node
// for which until(node) is true. During the traversal, if the Visit field is non-nil, it
// is called with the nodes joined by each followed edge.
func (d *DepthFirst) Walk(g graph.Graph, from graph.Node, until func(graph.Node) bool) graph.Node {
if d.visited == nil {
d.visited = &intsets.Sparse{}
}
d.stack.Push(from)
d.visited.Insert(from.ID())
for d.stack.Len() > 0 {
t := d.stack.Pop()
if until != nil && until(t) {
return t
}
for _, n := range g.From(t) {
if d.EdgeFilter != nil && !d.EdgeFilter(g.Edge(t, n)) {
continue
}
if d.visited.Has(n.ID()) {
continue
}
if d.Visit != nil {
d.Visit(t, n)
}
d.visited.Insert(n.ID())
d.stack.Push(n)
}
}
return nil
}
// WalkAll calls Walk for each unvisited node of the graph g using edges independent
// of their direction. The functions before and after are called prior to commencing
// and after completing each walk if they are non-nil respectively. The function
// during is called on each node as it is traversed.
func (d *DepthFirst) WalkAll(g graph.Undirected, before, after func(), during func(graph.Node)) {
d.Reset()
for _, from := range g.Nodes() {
if d.Visited(from) {
continue
}
if before != nil {
before()
}
d.Walk(g, from, func(n graph.Node) bool {
if during != nil {
during(n)
}
return false
})
if after != nil {
after()
}
}
}
// Visited returned whether the node n was visited during a traverse.
func (d *DepthFirst) Visited(n graph.Node) bool {
return d.visited != nil && d.visited.Has(n.ID())
}
// Reset resets the state of the traverser for reuse.
func (d *DepthFirst) Reset() {
d.stack = d.stack[:0]
if d.visited != nil {
d.visited.Clear()
}
}

View file

@ -1,86 +0,0 @@
// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package traverse provides basic graph traversal primitives.
package traverse
import (
"golang.org/x/tools/container/intsets"
"k8s.io/kubernetes/third_party/forked/gonum/graph"
"k8s.io/kubernetes/third_party/forked/gonum/graph/internal/linear"
)
// VisitableGraph
type VisitableGraph interface {
graph.Graph
// VisitFrom invokes visitor with all nodes that can be reached directly from the given node.
// If visitor returns false, visiting is short-circuited.
VisitFrom(from graph.Node, visitor func(graph.Node) (shouldContinue bool))
}
// VisitingDepthFirst implements stateful depth-first graph traversal on a visitable graph.
type VisitingDepthFirst struct {
EdgeFilter func(graph.Edge) bool
Visit func(u, v graph.Node)
stack linear.NodeStack
visited *intsets.Sparse
}
// Walk performs a depth-first traversal of the graph g starting from the given node,
// depending on the the EdgeFilter field and the until parameter if they are non-nil. The
// traversal follows edges for which EdgeFilter(edge) is true and returns the first node
// for which until(node) is true. During the traversal, if the Visit field is non-nil, it
// is called with the nodes joined by each followed edge.
func (d *VisitingDepthFirst) Walk(g VisitableGraph, from graph.Node, until func(graph.Node) bool) graph.Node {
if d.visited == nil {
d.visited = &intsets.Sparse{}
}
d.stack.Push(from)
d.visited.Insert(from.ID())
if until != nil && until(from) {
return from
}
var found graph.Node
for d.stack.Len() > 0 {
t := d.stack.Pop()
g.VisitFrom(t, func(n graph.Node) (shouldContinue bool) {
if d.EdgeFilter != nil && !d.EdgeFilter(g.Edge(t, n)) {
return true
}
if d.visited.Has(n.ID()) {
return true
}
if d.Visit != nil {
d.Visit(t, n)
}
d.visited.Insert(n.ID())
d.stack.Push(n)
if until != nil && until(n) {
found = n
return false
}
return true
})
if found != nil {
return found
}
}
return nil
}
// Visited returned whether the node n was visited during a traverse.
func (d *VisitingDepthFirst) Visited(n graph.Node) bool {
return d.visited != nil && d.visited.Has(n.ID())
}
// Reset resets the state of the traverser for reuse.
func (d *VisitingDepthFirst) Reset() {
d.stack = d.stack[:0]
if d.visited != nil {
d.visited.Clear()
}
}

View file

@ -1,172 +0,0 @@
Shell2Junit License Information
Feb, 2010
shell2junit library and sample code is licensed under Apache License, v.2.0.
(c) 2009 Manolo Carrasco (Manuel Carrasco Moñino)
=====
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the
copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other
entities that control, are controlled by, or are under common control with
that entity. For the purposes of this definition, "control" means (i) the
power, direct or indirect, to cause the direction or management of such
entity, whether by contract or otherwise, or (ii) ownership of fifty percent
(50%) or more of the outstanding shares, or (iii) beneficial ownership of
such entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising
permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation source, and
configuration files.
"Object" form shall mean any form resulting from mechanical transformation
or translation of a Source form, including but not limited to compiled
object code, generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form,
made available under the License, as indicated by a copyright notice that is
included in or attached to the work (an example is provided in the Appendix
below).
"Derivative Works" shall mean any work, whether in Source or Object form,
that is based on (or derived from) the Work and for which the editorial
revisions, annotations, elaborations, or other modifications represent, as a
whole, an original work of authorship. For the purposes of this License,
Derivative Works shall not include works that remain separable from, or
merely link (or bind by name) to the interfaces of, the Work and Derivative
Works thereof.
"Contribution" shall mean any work of authorship, including the original
version of the Work and any modifications or additions to that Work or
Derivative Works thereof, that is intentionally submitted to Licensor for
inclusion in the Work by the copyright owner or by an individual or Legal
Entity authorized to submit on behalf of the copyright owner. For the
purposes of this definition, "submitted" means any form of electronic,
verbal, or written communication sent to the Licensor or its
representatives, including but not limited to communication on electronic
mailing lists, source code control systems, and issue tracking systems that
are managed by, or on behalf of, the Licensor for the purpose of discussing
and improving the Work, but excluding communication that is conspicuously
marked or otherwise designated in writing by the copyright owner as "Not a
Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity on
behalf of whom a Contribution has been received by Licensor and subsequently
incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of this
License, each Contributor hereby grants to You a perpetual, worldwide,
non-exclusive, no-charge, royalty-free, irrevocable copyright license to
reproduce, prepare Derivative Works of, publicly display, publicly perform,
sublicense, and distribute the Work and such Derivative Works in Source or
Object form.
3. Grant of Patent License. Subject to the terms and conditions of this
License, each Contributor hereby grants to You a perpetual, worldwide,
non-exclusive, no-charge, royalty-free, irrevocable (except as stated in
this section) patent license to make, have made, use, offer to sell, sell,
import, and otherwise transfer the Work, where such license applies only to
those patent claims licensable by such Contributor that are necessarily
infringed by their Contribution(s) alone or by combination of their
Contribution(s) with the Work to which such Contribution(s) was submitted.
If You institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
Contribution incorporated within the Work constitutes direct or contributory
patent infringement, then any patent licenses granted to You under this
License for that Work shall terminate as of the date such litigation is
filed.
4. Redistribution. You may reproduce and distribute copies of the Work or
Derivative Works thereof in any medium, with or without modifications, and
in Source or Object form, provided that You meet the following conditions:
a. You must give any other recipients of the Work or Derivative Works a copy
of this License; and
b. You must cause any modified files to carry prominent notices stating that
You changed the files; and
c. You must retain, in the Source form of any Derivative Works that You
distribute, all copyright, patent, trademark, and attribution notices from
the Source form of the Work, excluding those notices that do not pertain to
any part of the Derivative Works; and
d. If the Work includes a "NOTICE" text file as part of its distribution,
then any Derivative Works that You distribute must include a readable copy
of the attribution notices contained within such NOTICE file, excluding
those notices that do not pertain to any part of the Derivative Works, in at
least one of the following places: within a NOTICE text file distributed as
part of the Derivative Works; within the Source form or documentation, if
provided along with the Derivative Works; or, within a display generated by
the Derivative Works, if and wherever such third-party notices normally
appear. The contents of the NOTICE file are for informational purposes only
and do not modify the License. You may add Your own attribution notices
within Derivative Works that You distribute, alongside or as an addendum to
the NOTICE text from the Work, provided that such additional attribution
notices cannot be construed as modifying the License.
You may add Your own copyright statement to Your modifications and may
provide additional or different license terms and conditions for use,
reproduction, or distribution of Your modifications, or for any such
Derivative Works as a whole, provided Your use, reproduction, and
distribution of the Work otherwise complies with the conditions stated in
this License.
5. Submission of Contributions. Unless You explicitly state otherwise, any
Contribution intentionally submitted for inclusion in the Work by You to the
Licensor shall be under the terms and conditions of this License, without
any additional terms or conditions. Notwithstanding the above, nothing
herein shall supersede or modify the terms of any separate license agreement
you may have executed with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor, except
as required for reasonable and customary use in describing the origin of the
Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or agreed to in
writing, Licensor provides the Work (and each Contributor provides its
Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied, including, without limitation, any
warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or
FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining
the appropriateness of using or redistributing the Work and assume any risks
associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory, whether
in tort (including negligence), contract, or otherwise, unless required by
applicable law (such as deliberate and grossly negligent acts) or agreed to
in writing, shall any Contributor be liable to You for damages, including
any direct, indirect, special, incidental, or consequential damages of any
character arising as a result of this License or out of the use or inability
to use the Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all other
commercial damages or losses), even if such Contributor has been advised of
the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing the Work
or Derivative Works thereof, You may choose to offer, and charge a fee for,
acceptance of support, warranty, indemnity, or other liability obligations
and/or rights consistent with this License. However, in accepting such
obligations, You may act only on Your own behalf and on Your sole
responsibility, not on behalf of any other Contributor, and only if You
agree to indemnify, defend, and hold each Contributor harmless for any
liability incurred by, or claims asserted against, such Contributor by
reason of your accepting any such warranty or additional liability.

View file

@ -1,8 +0,0 @@
reviewers:
- ixdy
- mengqiy
- pwittrock
approvers:
- ixdy
- mengqiy
- pwittrock

View file

@ -1,178 +0,0 @@
#!/bin/bash
### Copyright 2010 Manuel Carrasco Moñino. (manolo at apache.org)
###
### Licensed under the Apache License, Version 2.0.
### You may obtain a copy of it at
### http://www.apache.org/licenses/LICENSE-2.0
###
### A library for shell scripts which creates reports in jUnit format.
### These reports can be used in Jenkins, or any other CI.
###
### Usage:
### - Include this file in your shell script
### - Use juLog to call your command any time you want to produce a new report
### Usage: juLog <options> command arguments
### options:
### -class="MyClass" : a class name which will be shown in the junit report
### -name="TestName" : the test name which will be shown in the junit report
### -error="RegExp" : a regexp which sets the test as failure when the output matches it
### -ierror="RegExp" : same as -error but case insensitive
### -output="Path" : path to output directory, defaults to "./results"
### - Junit reports are left in the folder 'result' under the directory where the script is executed.
### - Configure Jenkins to parse junit files from the generated folder
###
asserts=00; errors=0; total=0; content=""
date=`which gdate 2>/dev/null || which date`
# default output folder
juDIR=`pwd`/results
# The name of the suite is calculated based in your script name
suite=""
# A wrapper for the eval method witch allows catching seg-faults and use tee
errfile=/tmp/evErr.$$.log
function eVal() {
(eval "$1")
# stdout and stderr may currently be inverted (see below) so echo may write to stderr
echo $? 2>&1 | tr -d "\n" > $errfile
}
# Method to clean old tests
function juLogClean() {
echo "+++ Removing old junit reports from: $juDIR "
rm -f "$juDIR"/junit-*
}
# Execute a command and record its results
function juLog() {
suite="";
errfile=/tmp/evErr.$$.log
date=`which gdate || which date`
asserts=00; errors=0; total=0; content=""
# parse arguments
ya=""; icase=""
while [ -z "$ya" ]; do
case "$1" in
-name=*) name=`echo "$1" | sed -e 's/-name=//'`; shift;;
-class=*) class=`echo "$1" | sed -e 's/-class=//'`; shift;;
-ierror=*) ereg=`echo "$1" | sed -e 's/-ierror=//'`; icase="-i"; shift;;
-error=*) ereg=`echo "$1" | sed -e 's/-error=//'`; shift;;
-output=*) juDIR=`echo "$1" | sed -e 's/-output=//'`; shift;;
*) ya=1;;
esac
done
# create output directory
mkdir -p "$juDIR" || exit
# use first arg as name if it was not given
if [ -z "$name" ]; then
name="$asserts-$1"
shift
fi
if [[ "$class" = "" ]]; then
class="default"
fi
suite=$class
# calculate command to eval
[ -z "$1" ] && return
cmd="$1"; shift
while [ -n "$1" ]
do
cmd="$cmd \"$1\""
shift
done
# eval the command sending output to a file
outf=/var/tmp/ju$$.txt
errf=/var/tmp/ju$$-err.txt
>$outf
echo "" | tee -a $outf
echo "+++ Running case: $class.$name " | tee -a $outf
echo "+++ working dir: "`pwd` | tee -a $outf
echo "+++ command: $cmd" | tee -a $outf
ini=`$date +%s.%N`
# execute the command, temporarily swapping stderr and stdout so they can be tee'd to separate files,
# then swapping them back again so that the streams are written correctly for the invoking process
( (eVal "$cmd" | tee -a $outf) 3>&1 1>&2 2>&3 | tee $errf) 3>&1 1>&2 2>&3
evErr=`cat $errfile`
rm -f $errfile
end=`$date +%s.%N`
echo "+++ exit code: $evErr" | tee -a $outf
# set the appropriate error, based in the exit code and the regex
[ $evErr != 0 ] && err=1 || err=0
out=`cat $outf | sed -e 's/^\([^+]\)/| \1/g'`
if [ $err = 0 -a -n "$ereg" ]; then
H=`echo "$out" | egrep $icase "$ereg"`
[ -n "$H" ] && err=1
fi
[ $err != 0 ] && echo "+++ error: $err" | tee -a $outf
rm -f $outf
errMsg=`cat $errf`
rm -f $errf
# calculate vars
asserts=$(($asserts+1))
errors=$(($errors+$err))
time=`echo "$end - $ini" | bc -l`
total=`echo "$total + $time" | bc -l`
# write the junit xml report
## failure tag
[ $err = 0 ] && failure="" || failure="
<failure type=\"ScriptError\" message=\"Script Error\">
<![CDATA[
$errMsg
]]>
</failure>
"
## testcase tag
content="$content
<testcase assertions=\"1\" name=\"$name\" time=\"$time\" classname=\"$class\">
$failure
<system-err>
<![CDATA[
$errMsg
]]>
</system-err>
</testcase>
"
## testsuite block
if [[ -e "$juDIR/junit_$suite.xml" ]]; then
# file exists. first update the failures count
failCount=`sed -n "s/.*testsuite.*failures=\"\([0-9]*\)\".*/\1/p" "$juDIR/junit_$suite.xml"`
errors=$(($failCount+$errors))
sed -i "0,/failures=\"$failCount\"/ s/failures=\"$failCount\"/failures=\"$errors\"/" "$juDIR/junit_$suite.xml"
sed -i "0,/errors=\"$failCount\"/ s/errors=\"$failCount\"/errors=\"$errors\"/" "$juDIR/junit_$suite.xml"
# file exists. Need to append to it. If we remove the testsuite end tag, we can just add it in after.
sed -i "s^</testsuite>^^g" $juDIR/junit_$suite.xml ## remove testSuite so we can add it later
sed -i "s^</testsuites>^^g" $juDIR/junit_$suite.xml
cat <<EOF >> "$juDIR/junit_$suite.xml"
$content
</testsuite>
</testsuites>
EOF
else
# no file exists. Adding a new file
cat <<EOF > "$juDIR/junit_$suite.xml"
<?xml version="1.0" encoding="UTF-8"?>
<testsuites>
<testsuite failures="$errors" assertions="$assertions" name="$suite" tests="1" errors="$errors" time="$total">
$content
</testsuite>
</testsuites>
EOF
fi
return $err
}

View file

@ -1,16 +0,0 @@
package(default_visibility = ["//visibility:public"])
licenses(["notice"])
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
)

View file

@ -1,28 +0,0 @@
Copyright (C) 2003-2013 Edgewall Software
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in
the documentation and/or other materials provided with the
distribution.
3. The name of the author may not be used to endorse or promote
products derived from this software without specific prior
written permission.
THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS
OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE
GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

View file

@ -1,153 +0,0 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# Copyright (C) 2008-2013 Edgewall Software
# Copyright (C) 2008 Eli Carter
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.com/license.html.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/.
"""Replacement for htpasswd"""
import os
import sys
import random
from optparse import OptionParser
# We need a crypt module, but Windows doesn't have one by default. Try to find
# one, and tell the user if we can't.
try:
import crypt
except ImportError:
try:
import fcrypt as crypt
except ImportError:
sys.stderr.write("Cannot find a crypt module. "
"Possibly http://carey.geek.nz/code/python-fcrypt/\n")
sys.exit(1)
def wait_for_file_mtime_change(filename):
"""This function is typically called before a file save operation,
waiting if necessary for the file modification time to change. The
purpose is to avoid successive file updates going undetected by the
caching mechanism that depends on a change in the file modification
time to know when the file should be reparsed."""
try:
mtime = os.stat(filename).st_mtime
os.utime(filename, None)
while mtime == os.stat(filename).st_mtime:
time.sleep(1e-3)
os.utime(filename, None)
except OSError:
pass # file doesn't exist (yet)
def salt():
"""Returns a string of 2 randome letters"""
letters = 'abcdefghijklmnopqrstuvwxyz' \
'ABCDEFGHIJKLMNOPQRSTUVWXYZ' \
'0123456789/.'
return random.choice(letters) + random.choice(letters)
class HtpasswdFile:
"""A class for manipulating htpasswd files."""
def __init__(self, filename, create=False):
self.entries = []
self.filename = filename
if not create:
if os.path.exists(self.filename):
self.load()
else:
raise Exception("%s does not exist" % self.filename)
def load(self):
"""Read the htpasswd file into memory."""
lines = open(self.filename, 'r').readlines()
self.entries = []
for line in lines:
username, pwhash = line.split(':')
entry = [username, pwhash.rstrip()]
self.entries.append(entry)
def save(self):
"""Write the htpasswd file to disk"""
wait_for_file_mtime_change(self.filename)
open(self.filename, 'w').writelines(["%s:%s\n" % (entry[0], entry[1])
for entry in self.entries])
def update(self, username, password):
"""Replace the entry for the given user, or add it if new."""
pwhash = crypt.crypt(password, salt())
matching_entries = [entry for entry in self.entries
if entry[0] == username]
if matching_entries:
matching_entries[0][1] = pwhash
else:
self.entries.append([username, pwhash])
def delete(self, username):
"""Remove the entry for the given user."""
self.entries = [entry for entry in self.entries
if entry[0] != username]
def main():
"""
%prog -b[c] filename username password
%prog -D filename username"""
# For now, we only care about the use cases that affect tests/functional.py
parser = OptionParser(usage=main.__doc__)
parser.add_option('-b', action='store_true', dest='batch', default=False,
help='Batch mode; password is passed on the command line IN THE CLEAR.'
)
parser.add_option('-c', action='store_true', dest='create', default=False,
help='Create a new htpasswd file, overwriting any existing file.')
parser.add_option('-D', action='store_true', dest='delete_user',
default=False, help='Remove the given user from the password file.')
options, args = parser.parse_args()
def syntax_error(msg):
"""Utility function for displaying fatal error messages with usage
help.
"""
sys.stderr.write("Syntax error: " + msg)
sys.stderr.write(parser.get_usage())
sys.exit(1)
if not (options.batch or options.delete_user):
syntax_error("Only batch and delete modes are supported\n")
# Non-option arguments
if len(args) < 2:
syntax_error("Insufficient number of arguments.\n")
filename, username = args[:2]
if options.delete_user:
if len(args) != 2:
syntax_error("Incorrect number of arguments.\n")
password = None
else:
if len(args) != 3:
syntax_error("Incorrect number of arguments.\n")
password = args[2]
passwdfile = HtpasswdFile(filename, create=options.create)
if options.delete_user:
passwdfile.delete(username)
else:
passwdfile.update(username, password)
passwdfile.save()
if __name__ == '__main__':
main()

View file

@ -1,202 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View file

@ -1,12 +0,0 @@
prefix=/usr/local/bin
default: build
build:
@echo "Nothing to build. Use make install"
install: intemp.sh
install intemp.sh $(DESTDIR)$(prefix)
uninstall:
-rm $(DESTDIR)$(prefix)/intemp.sh

View file

@ -1,52 +0,0 @@
# intemp
A bash script to execute a command within a temporary work directory.
## Dependencies
Requires: mktemp
## Install
```
git clone https://github.com/karlkfi/intemp
cd intemp
make install
```
or
```
curl -o- https://raw.githubusercontent.com/karlkfi/intemp/master/install.sh | bash
```
## Usage
```
intemp.sh [-t prefix] "<command>"
```
Example (install intemp using intemp):
```
intemp.sh -t intemp "git clone https://github.com/karlkfi/intemp . && make install"
```
## License
Copyright 2015 Karl Isenberg
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View file

@ -1,21 +0,0 @@
#!/usr/bin/env bash
# Install intemp from the internet
# Usage: install.sh [version]
# Alt Usage: curl -o- https://raw.githubusercontent.com/karlkfi/intemp/master/install.sh | bash
# Requires: curl
set -o errexit
set -o nounset
set -o pipefail
prefix="/usr/local/bin"
version=${1:-}
if [ -z "${version}" ]; then
version=$(curl -s https://api.github.com/repos/karlkfi/intemp/releases/latest | grep 'tag_name' | cut -d\" -f4)
fi
echo "Installing intemp ${version} -> ${prefix}/intemp.sh"
curl -o- "https://raw.githubusercontent.com/karlkfi/intemp/${version}/intemp.sh" > "${prefix}/intemp.sh"
chmod a+x "${prefix}/intemp.sh"

View file

@ -1,41 +0,0 @@
#!/usr/bin/env bash
# Runs the supplied bash command string in a temporary workspace directory.
# Usage: intemp.sh [-t prefix] <command>
# Requires: mktemp
set -o errexit
set -o nounset
set -o pipefail
opt_flag=${1:-}
[ -z "${opt_flag}" ] && echo "No command supplied" >&2 && exit 1
if [ "${opt_flag}" == "-t" ]; then
shift
prefix=${1:-}
[ -z "${prefix}" ] && echo "No prefix supplied" >&2 && exit 1
shift
else
prefix='temp'
fi
cmd="$1"
[ -z "${cmd}" ] && echo "No command supplied" >&2 && exit 1
workspace=$(mktemp -d "${TMPDIR:-/tmp}/${prefix}.XXXXXX")
echo "Workspace created: ${workspace}" 1>&2
cleanup() {
local -r workspace="$1"
rm -rf "${workspace}"
echo "Workspace deleted: ${workspace}" 1>&2
}
trap "cleanup '${workspace}'" EXIT
pushd "${workspace}" > /dev/null
bash -ceu "${cmd}"
popd > /dev/null
trap - EXIT
cleanup "${workspace}"

View file

@ -1,150 +0,0 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: kenton@google.com (Kenton Varda)
//
// WARNING: The plugin interface is currently EXPERIMENTAL and is subject to
// change.
//
// protoc (aka the Protocol Compiler) can be extended via plugins. A plugin is
// just a program that reads a CodeGeneratorRequest from stdin and writes a
// CodeGeneratorResponse to stdout.
//
// Plugins written using C++ can use google/protobuf/compiler/plugin.h instead
// of dealing with the raw protocol defined here.
//
// A plugin executable needs only to be placed somewhere in the path. The
// plugin should be named "protoc-gen-$NAME", and will then be used when the
// flag "--${NAME}_out" is passed to protoc.
syntax = "proto2";
package google.protobuf.compiler;
option java_package = "com.google.protobuf.compiler";
option java_outer_classname = "PluginProtos";
option go_package = "plugin_go";
import "google/protobuf/descriptor.proto";
// An encoded CodeGeneratorRequest is written to the plugin's stdin.
message CodeGeneratorRequest {
// The .proto files that were explicitly listed on the command-line. The
// code generator should generate code only for these files. Each file's
// descriptor will be included in proto_file, below.
repeated string file_to_generate = 1;
// The generator parameter passed on the command-line.
optional string parameter = 2;
// FileDescriptorProtos for all files in files_to_generate and everything
// they import. The files will appear in topological order, so each file
// appears before any file that imports it.
//
// protoc guarantees that all proto_files will be written after
// the fields above, even though this is not technically guaranteed by the
// protobuf wire format. This theoretically could allow a plugin to stream
// in the FileDescriptorProtos and handle them one by one rather than read
// the entire set into memory at once. However, as of this writing, this
// is not similarly optimized on protoc's end -- it will store all fields in
// memory at once before sending them to the plugin.
repeated FileDescriptorProto proto_file = 15;
}
// The plugin writes an encoded CodeGeneratorResponse to stdout.
message CodeGeneratorResponse {
// Error message. If non-empty, code generation failed. The plugin process
// should exit with status code zero even if it reports an error in this way.
//
// This should be used to indicate errors in .proto files which prevent the
// code generator from generating correct code. Errors which indicate a
// problem in protoc itself -- such as the input CodeGeneratorRequest being
// unparseable -- should be reported by writing a message to stderr and
// exiting with a non-zero status code.
optional string error = 1;
// Represents a single generated file.
message File {
// The file name, relative to the output directory. The name must not
// contain "." or ".." components and must be relative, not be absolute (so,
// the file cannot lie outside the output directory). "/" must be used as
// the path separator, not "\".
//
// If the name is omitted, the content will be appended to the previous
// file. This allows the generator to break large files into small chunks,
// and allows the generated text to be streamed back to protoc so that large
// files need not reside completely in memory at one time. Note that as of
// this writing protoc does not optimize for this -- it will read the entire
// CodeGeneratorResponse before writing files to disk.
optional string name = 1;
// If non-empty, indicates that the named file should already exist, and the
// content here is to be inserted into that file at a defined insertion
// point. This feature allows a code generator to extend the output
// produced by another code generator. The original generator may provide
// insertion points by placing special annotations in the file that look
// like:
// @@protoc_insertion_point(NAME)
// The annotation can have arbitrary text before and after it on the line,
// which allows it to be placed in a comment. NAME should be replaced with
// an identifier naming the point -- this is what other generators will use
// as the insertion_point. Code inserted at this point will be placed
// immediately above the line containing the insertion point (thus multiple
// insertions to the same point will come out in the order they were added).
// The double-@ is intended to make it unlikely that the generated code
// could contain things that look like insertion points by accident.
//
// For example, the C++ code generator places the following line in the
// .pb.h files that it generates:
// // @@protoc_insertion_point(namespace_scope)
// This line appears within the scope of the file's package namespace, but
// outside of any particular class. Another plugin can then specify the
// insertion_point "namespace_scope" to generate additional classes or
// other declarations that should be placed in this scope.
//
// Note that if the line containing the insertion point begins with
// whitespace, the same whitespace will be added to every line of the
// inserted text. This is useful for languages like Python, where
// indentation matters. In these languages, the insertion point comment
// should be indented the same amount as any inserted code will need to be
// in order to work correctly in that context.
//
// The code generator that generates the initial file and the one which
// inserts into it must both run as part of a single invocation of protoc.
// Code generators are executed in the order in which they appear on the
// command line.
//
// If |insertion_point| is present, |name| must also be present.
optional string insertion_point = 2;
// The file contents.
optional string content = 15;
}
repeated File file = 15;
}

View file

@ -1,779 +0,0 @@
// Protocol Buffers - Google's data interchange format
// Copyright 2008 Google Inc. All rights reserved.
// https://developers.google.com/protocol-buffers/
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// Author: kenton@google.com (Kenton Varda)
// Based on original Protocol Buffers design by
// Sanjay Ghemawat, Jeff Dean, and others.
//
// The messages in this file describe the definitions found in .proto files.
// A valid .proto file can be translated directly to a FileDescriptorProto
// without any other information (e.g. without reading its imports).
syntax = "proto2";
package google.protobuf;
option go_package = "descriptor";
option java_package = "com.google.protobuf";
option java_outer_classname = "DescriptorProtos";
option csharp_namespace = "Google.Protobuf.Reflection";
option objc_class_prefix = "GPB";
// descriptor.proto must be optimized for speed because reflection-based
// algorithms don't work during bootstrapping.
option optimize_for = SPEED;
// The protocol compiler can output a FileDescriptorSet containing the .proto
// files it parses.
message FileDescriptorSet {
repeated FileDescriptorProto file = 1;
}
// Describes a complete .proto file.
message FileDescriptorProto {
optional string name = 1; // file name, relative to root of source tree
optional string package = 2; // e.g. "foo", "foo.bar", etc.
// Names of files imported by this file.
repeated string dependency = 3;
// Indexes of the public imported files in the dependency list above.
repeated int32 public_dependency = 10;
// Indexes of the weak imported files in the dependency list.
// For Google-internal migration only. Do not use.
repeated int32 weak_dependency = 11;
// All top-level definitions in this file.
repeated DescriptorProto message_type = 4;
repeated EnumDescriptorProto enum_type = 5;
repeated ServiceDescriptorProto service = 6;
repeated FieldDescriptorProto extension = 7;
optional FileOptions options = 8;
// This field contains optional information about the original source code.
// You may safely remove this entire field without harming runtime
// functionality of the descriptors -- the information is needed only by
// development tools.
optional SourceCodeInfo source_code_info = 9;
// The syntax of the proto file.
// The supported values are "proto2" and "proto3".
optional string syntax = 12;
}
// Describes a message type.
message DescriptorProto {
optional string name = 1;
repeated FieldDescriptorProto field = 2;
repeated FieldDescriptorProto extension = 6;
repeated DescriptorProto nested_type = 3;
repeated EnumDescriptorProto enum_type = 4;
message ExtensionRange {
optional int32 start = 1;
optional int32 end = 2;
}
repeated ExtensionRange extension_range = 5;
repeated OneofDescriptorProto oneof_decl = 8;
optional MessageOptions options = 7;
// Range of reserved tag numbers. Reserved tag numbers may not be used by
// fields or extension ranges in the same message. Reserved ranges may
// not overlap.
message ReservedRange {
optional int32 start = 1; // Inclusive.
optional int32 end = 2; // Exclusive.
}
repeated ReservedRange reserved_range = 9;
// Reserved field names, which may not be used by fields in the same message.
// A given name may only be reserved once.
repeated string reserved_name = 10;
}
// Describes a field within a message.
message FieldDescriptorProto {
enum Type {
// 0 is reserved for errors.
// Order is weird for historical reasons.
TYPE_DOUBLE = 1;
TYPE_FLOAT = 2;
// Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT64 if
// negative values are likely.
TYPE_INT64 = 3;
TYPE_UINT64 = 4;
// Not ZigZag encoded. Negative numbers take 10 bytes. Use TYPE_SINT32 if
// negative values are likely.
TYPE_INT32 = 5;
TYPE_FIXED64 = 6;
TYPE_FIXED32 = 7;
TYPE_BOOL = 8;
TYPE_STRING = 9;
TYPE_GROUP = 10; // Tag-delimited aggregate.
TYPE_MESSAGE = 11; // Length-delimited aggregate.
// New in version 2.
TYPE_BYTES = 12;
TYPE_UINT32 = 13;
TYPE_ENUM = 14;
TYPE_SFIXED32 = 15;
TYPE_SFIXED64 = 16;
TYPE_SINT32 = 17; // Uses ZigZag encoding.
TYPE_SINT64 = 18; // Uses ZigZag encoding.
};
enum Label {
// 0 is reserved for errors
LABEL_OPTIONAL = 1;
LABEL_REQUIRED = 2;
LABEL_REPEATED = 3;
// TODO(sanjay): Should we add LABEL_MAP?
};
optional string name = 1;
optional int32 number = 3;
optional Label label = 4;
// If type_name is set, this need not be set. If both this and type_name
// are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP.
optional Type type = 5;
// For message and enum types, this is the name of the type. If the name
// starts with a '.', it is fully-qualified. Otherwise, C++-like scoping
// rules are used to find the type (i.e. first the nested types within this
// message are searched, then within the parent, on up to the root
// namespace).
optional string type_name = 6;
// For extensions, this is the name of the type being extended. It is
// resolved in the same manner as type_name.
optional string extendee = 2;
// For numeric types, contains the original text representation of the value.
// For booleans, "true" or "false".
// For strings, contains the default text contents (not escaped in any way).
// For bytes, contains the C escaped value. All bytes >= 128 are escaped.
// TODO(kenton): Base-64 encode?
optional string default_value = 7;
// If set, gives the index of a oneof in the containing type's oneof_decl
// list. This field is a member of that oneof.
optional int32 oneof_index = 9;
// JSON name of this field. The value is set by protocol compiler. If the
// user has set a "json_name" option on this field, that option's value
// will be used. Otherwise, it's deduced from the field's name by converting
// it to camelCase.
optional string json_name = 10;
optional FieldOptions options = 8;
}
// Describes a oneof.
message OneofDescriptorProto {
optional string name = 1;
}
// Describes an enum type.
message EnumDescriptorProto {
optional string name = 1;
repeated EnumValueDescriptorProto value = 2;
optional EnumOptions options = 3;
}
// Describes a value within an enum.
message EnumValueDescriptorProto {
optional string name = 1;
optional int32 number = 2;
optional EnumValueOptions options = 3;
}
// Describes a service.
message ServiceDescriptorProto {
optional string name = 1;
repeated MethodDescriptorProto method = 2;
optional ServiceOptions options = 3;
}
// Describes a method of a service.
message MethodDescriptorProto {
optional string name = 1;
// Input and output type names. These are resolved in the same way as
// FieldDescriptorProto.type_name, but must refer to a message type.
optional string input_type = 2;
optional string output_type = 3;
optional MethodOptions options = 4;
// Identifies if client streams multiple client messages
optional bool client_streaming = 5 [default=false];
// Identifies if server streams multiple server messages
optional bool server_streaming = 6 [default=false];
}
// ===================================================================
// Options
// Each of the definitions above may have "options" attached. These are
// just annotations which may cause code to be generated slightly differently
// or may contain hints for code that manipulates protocol messages.
//
// Clients may define custom options as extensions of the *Options messages.
// These extensions may not yet be known at parsing time, so the parser cannot
// store the values in them. Instead it stores them in a field in the *Options
// message called uninterpreted_option. This field must have the same name
// across all *Options messages. We then use this field to populate the
// extensions when we build a descriptor, at which point all protos have been
// parsed and so all extensions are known.
//
// Extension numbers for custom options may be chosen as follows:
// * For options which will only be used within a single application or
// organization, or for experimental options, use field numbers 50000
// through 99999. It is up to you to ensure that you do not use the
// same number for multiple options.
// * For options which will be published and used publicly by multiple
// independent entities, e-mail protobuf-global-extension-registry@google.com
// to reserve extension numbers. Simply provide your project name (e.g.
// Objective-C plugin) and your project website (if available) -- there's no
// need to explain how you intend to use them. Usually you only need one
// extension number. You can declare multiple options with only one extension
// number by putting them in a sub-message. See the Custom Options section of
// the docs for examples:
// https://developers.google.com/protocol-buffers/docs/proto#options
// If this turns out to be popular, a web service will be set up
// to automatically assign option numbers.
message FileOptions {
// Sets the Java package where classes generated from this .proto will be
// placed. By default, the proto package is used, but this is often
// inappropriate because proto packages do not normally start with backwards
// domain names.
optional string java_package = 1;
// If set, all the classes from the .proto file are wrapped in a single
// outer class with the given name. This applies to both Proto1
// (equivalent to the old "--one_java_file" option) and Proto2 (where
// a .proto always translates to a single class, but you may want to
// explicitly choose the class name).
optional string java_outer_classname = 8;
// If set true, then the Java code generator will generate a separate .java
// file for each top-level message, enum, and service defined in the .proto
// file. Thus, these types will *not* be nested inside the outer class
// named by java_outer_classname. However, the outer class will still be
// generated to contain the file's getDescriptor() method as well as any
// top-level extensions defined in the file.
optional bool java_multiple_files = 10 [default=false];
// If set true, then the Java code generator will generate equals() and
// hashCode() methods for all messages defined in the .proto file.
// This increases generated code size, potentially substantially for large
// protos, which may harm a memory-constrained application.
// - In the full runtime this is a speed optimization, as the
// AbstractMessage base class includes reflection-based implementations of
// these methods.
// - In the lite runtime, setting this option changes the semantics of
// equals() and hashCode() to more closely match those of the full runtime;
// the generated methods compute their results based on field values rather
// than object identity. (Implementations should not assume that hashcodes
// will be consistent across runtimes or versions of the protocol compiler.)
optional bool java_generate_equals_and_hash = 20 [default=false];
// If set true, then the Java2 code generator will generate code that
// throws an exception whenever an attempt is made to assign a non-UTF-8
// byte sequence to a string field.
// Message reflection will do the same.
// However, an extension field still accepts non-UTF-8 byte sequences.
// This option has no effect on when used with the lite runtime.
optional bool java_string_check_utf8 = 27 [default=false];
// Generated classes can be optimized for speed or code size.
enum OptimizeMode {
SPEED = 1; // Generate complete code for parsing, serialization,
// etc.
CODE_SIZE = 2; // Use ReflectionOps to implement these methods.
LITE_RUNTIME = 3; // Generate code using MessageLite and the lite runtime.
}
optional OptimizeMode optimize_for = 9 [default=SPEED];
// Sets the Go package where structs generated from this .proto will be
// placed. If omitted, the Go package will be derived from the following:
// - The basename of the package import path, if provided.
// - Otherwise, the package statement in the .proto file, if present.
// - Otherwise, the basename of the .proto file, without extension.
optional string go_package = 11;
// Should generic services be generated in each language? "Generic" services
// are not specific to any particular RPC system. They are generated by the
// main code generators in each language (without additional plugins).
// Generic services were the only kind of service generation supported by
// early versions of google.protobuf.
//
// Generic services are now considered deprecated in favor of using plugins
// that generate code specific to your particular RPC system. Therefore,
// these default to false. Old code which depends on generic services should
// explicitly set them to true.
optional bool cc_generic_services = 16 [default=false];
optional bool java_generic_services = 17 [default=false];
optional bool py_generic_services = 18 [default=false];
// Is this file deprecated?
// Depending on the target platform, this can emit Deprecated annotations
// for everything in the file, or it will be completely ignored; in the very
// least, this is a formalization for deprecating files.
optional bool deprecated = 23 [default=false];
// Enables the use of arenas for the proto messages in this file. This applies
// only to generated classes for C++.
optional bool cc_enable_arenas = 31 [default=false];
// Sets the objective c class prefix which is prepended to all objective c
// generated classes from this .proto. There is no default.
optional string objc_class_prefix = 36;
// Namespace for generated classes; defaults to the package.
optional string csharp_namespace = 37;
// Whether the nano proto compiler should generate in the deprecated non-nano
// suffixed package.
optional bool javanano_use_deprecated_package = 38;
// The parser stores options it doesn't recognize here. See above.
repeated UninterpretedOption uninterpreted_option = 999;
// Clients can define custom options in extensions of this message. See above.
extensions 1000 to max;
}
message MessageOptions {
// Set true to use the old proto1 MessageSet wire format for extensions.
// This is provided for backwards-compatibility with the MessageSet wire
// format. You should not use this for any other reason: It's less
// efficient, has fewer features, and is more complicated.
//
// The message must be defined exactly as follows:
// message Foo {
// option message_set_wire_format = true;
// extensions 4 to max;
// }
// Note that the message cannot have any defined fields; MessageSets only
// have extensions.
//
// All extensions of your type must be singular messages; e.g. they cannot
// be int32s, enums, or repeated messages.
//
// Because this is an option, the above two restrictions are not enforced by
// the protocol compiler.
optional bool message_set_wire_format = 1 [default=false];
// Disables the generation of the standard "descriptor()" accessor, which can
// conflict with a field of the same name. This is meant to make migration
// from proto1 easier; new code should avoid fields named "descriptor".
optional bool no_standard_descriptor_accessor = 2 [default=false];
// Is this message deprecated?
// Depending on the target platform, this can emit Deprecated annotations
// for the message, or it will be completely ignored; in the very least,
// this is a formalization for deprecating messages.
optional bool deprecated = 3 [default=false];
// Whether the message is an automatically generated map entry type for the
// maps field.
//
// For maps fields:
// map<KeyType, ValueType> map_field = 1;
// The parsed descriptor looks like:
// message MapFieldEntry {
// option map_entry = true;
// optional KeyType key = 1;
// optional ValueType value = 2;
// }
// repeated MapFieldEntry map_field = 1;
//
// Implementations may choose not to generate the map_entry=true message, but
// use a native map in the target language to hold the keys and values.
// The reflection APIs in such implementions still need to work as
// if the field is a repeated message field.
//
// NOTE: Do not set the option in .proto files. Always use the maps syntax
// instead. The option should only be implicitly set by the proto compiler
// parser.
optional bool map_entry = 7;
// The parser stores options it doesn't recognize here. See above.
repeated UninterpretedOption uninterpreted_option = 999;
// Clients can define custom options in extensions of this message. See above.
extensions 1000 to max;
}
message FieldOptions {
// The ctype option instructs the C++ code generator to use a different
// representation of the field than it normally would. See the specific
// options below. This option is not yet implemented in the open source
// release -- sorry, we'll try to include it in a future version!
optional CType ctype = 1 [default = STRING];
enum CType {
// Default mode.
STRING = 0;
CORD = 1;
STRING_PIECE = 2;
}
// The packed option can be enabled for repeated primitive fields to enable
// a more efficient representation on the wire. Rather than repeatedly
// writing the tag and type for each element, the entire array is encoded as
// a single length-delimited blob. In proto3, only explicit setting it to
// false will avoid using packed encoding.
optional bool packed = 2;
// The jstype option determines the JavaScript type used for values of the
// field. The option is permitted only for 64 bit integral and fixed types
// (int64, uint64, sint64, fixed64, sfixed64). By default these types are
// represented as JavaScript strings. This avoids loss of precision that can
// happen when a large value is converted to a floating point JavaScript
// numbers. Specifying JS_NUMBER for the jstype causes the generated
// JavaScript code to use the JavaScript "number" type instead of strings.
// This option is an enum to permit additional types to be added,
// e.g. goog.math.Integer.
optional JSType jstype = 6 [default = JS_NORMAL];
enum JSType {
// Use the default type.
JS_NORMAL = 0;
// Use JavaScript strings.
JS_STRING = 1;
// Use JavaScript numbers.
JS_NUMBER = 2;
}
// Should this field be parsed lazily? Lazy applies only to message-type
// fields. It means that when the outer message is initially parsed, the
// inner message's contents will not be parsed but instead stored in encoded
// form. The inner message will actually be parsed when it is first accessed.
//
// This is only a hint. Implementations are free to choose whether to use
// eager or lazy parsing regardless of the value of this option. However,
// setting this option true suggests that the protocol author believes that
// using lazy parsing on this field is worth the additional bookkeeping
// overhead typically needed to implement it.
//
// This option does not affect the public interface of any generated code;
// all method signatures remain the same. Furthermore, thread-safety of the
// interface is not affected by this option; const methods remain safe to
// call from multiple threads concurrently, while non-const methods continue
// to require exclusive access.
//
//
// Note that implementations may choose not to check required fields within
// a lazy sub-message. That is, calling IsInitialized() on the outher message
// may return true even if the inner message has missing required fields.
// This is necessary because otherwise the inner message would have to be
// parsed in order to perform the check, defeating the purpose of lazy
// parsing. An implementation which chooses not to check required fields
// must be consistent about it. That is, for any particular sub-message, the
// implementation must either *always* check its required fields, or *never*
// check its required fields, regardless of whether or not the message has
// been parsed.
optional bool lazy = 5 [default=false];
// Is this field deprecated?
// Depending on the target platform, this can emit Deprecated annotations
// for accessors, or it will be completely ignored; in the very least, this
// is a formalization for deprecating fields.
optional bool deprecated = 3 [default=false];
// For Google-internal migration only. Do not use.
optional bool weak = 10 [default=false];
// The parser stores options it doesn't recognize here. See above.
repeated UninterpretedOption uninterpreted_option = 999;
// Clients can define custom options in extensions of this message. See above.
extensions 1000 to max;
}
message EnumOptions {
// Set this option to true to allow mapping different tag names to the same
// value.
optional bool allow_alias = 2;
// Is this enum deprecated?
// Depending on the target platform, this can emit Deprecated annotations
// for the enum, or it will be completely ignored; in the very least, this
// is a formalization for deprecating enums.
optional bool deprecated = 3 [default=false];
// The parser stores options it doesn't recognize here. See above.
repeated UninterpretedOption uninterpreted_option = 999;
// Clients can define custom options in extensions of this message. See above.
extensions 1000 to max;
}
message EnumValueOptions {
// Is this enum value deprecated?
// Depending on the target platform, this can emit Deprecated annotations
// for the enum value, or it will be completely ignored; in the very least,
// this is a formalization for deprecating enum values.
optional bool deprecated = 1 [default=false];
// The parser stores options it doesn't recognize here. See above.
repeated UninterpretedOption uninterpreted_option = 999;
// Clients can define custom options in extensions of this message. See above.
extensions 1000 to max;
}
message ServiceOptions {
// Note: Field numbers 1 through 32 are reserved for Google's internal RPC
// framework. We apologize for hoarding these numbers to ourselves, but
// we were already using them long before we decided to release Protocol
// Buffers.
// Is this service deprecated?
// Depending on the target platform, this can emit Deprecated annotations
// for the service, or it will be completely ignored; in the very least,
// this is a formalization for deprecating services.
optional bool deprecated = 33 [default=false];
// The parser stores options it doesn't recognize here. See above.
repeated UninterpretedOption uninterpreted_option = 999;
// Clients can define custom options in extensions of this message. See above.
extensions 1000 to max;
}
message MethodOptions {
// Note: Field numbers 1 through 32 are reserved for Google's internal RPC
// framework. We apologize for hoarding these numbers to ourselves, but
// we were already using them long before we decided to release Protocol
// Buffers.
// Is this method deprecated?
// Depending on the target platform, this can emit Deprecated annotations
// for the method, or it will be completely ignored; in the very least,
// this is a formalization for deprecating methods.
optional bool deprecated = 33 [default=false];
// The parser stores options it doesn't recognize here. See above.
repeated UninterpretedOption uninterpreted_option = 999;
// Clients can define custom options in extensions of this message. See above.
extensions 1000 to max;
}
// A message representing a option the parser does not recognize. This only
// appears in options protos created by the compiler::Parser class.
// DescriptorPool resolves these when building Descriptor objects. Therefore,
// options protos in descriptor objects (e.g. returned by Descriptor::options(),
// or produced by Descriptor::CopyTo()) will never have UninterpretedOptions
// in them.
message UninterpretedOption {
// The name of the uninterpreted option. Each string represents a segment in
// a dot-separated name. is_extension is true iff a segment represents an
// extension (denoted with parentheses in options specs in .proto files).
// E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents
// "foo.(bar.baz).qux".
message NamePart {
required string name_part = 1;
required bool is_extension = 2;
}
repeated NamePart name = 2;
// The value of the uninterpreted option, in whatever type the tokenizer
// identified it as during parsing. Exactly one of these should be set.
optional string identifier_value = 3;
optional uint64 positive_int_value = 4;
optional int64 negative_int_value = 5;
optional double double_value = 6;
optional bytes string_value = 7;
optional string aggregate_value = 8;
}
// ===================================================================
// Optional source code info
// Encapsulates information about the original source file from which a
// FileDescriptorProto was generated.
message SourceCodeInfo {
// A Location identifies a piece of source code in a .proto file which
// corresponds to a particular definition. This information is intended
// to be useful to IDEs, code indexers, documentation generators, and similar
// tools.
//
// For example, say we have a file like:
// message Foo {
// optional string foo = 1;
// }
// Let's look at just the field definition:
// optional string foo = 1;
// ^ ^^ ^^ ^ ^^^
// a bc de f ghi
// We have the following locations:
// span path represents
// [a,i) [ 4, 0, 2, 0 ] The whole field definition.
// [a,b) [ 4, 0, 2, 0, 4 ] The label (optional).
// [c,d) [ 4, 0, 2, 0, 5 ] The type (string).
// [e,f) [ 4, 0, 2, 0, 1 ] The name (foo).
// [g,h) [ 4, 0, 2, 0, 3 ] The number (1).
//
// Notes:
// - A location may refer to a repeated field itself (i.e. not to any
// particular index within it). This is used whenever a set of elements are
// logically enclosed in a single code segment. For example, an entire
// extend block (possibly containing multiple extension definitions) will
// have an outer location whose path refers to the "extensions" repeated
// field without an index.
// - Multiple locations may have the same path. This happens when a single
// logical declaration is spread out across multiple places. The most
// obvious example is the "extend" block again -- there may be multiple
// extend blocks in the same scope, each of which will have the same path.
// - A location's span is not always a subset of its parent's span. For
// example, the "extendee" of an extension declaration appears at the
// beginning of the "extend" block and is shared by all extensions within
// the block.
// - Just because a location's span is a subset of some other location's span
// does not mean that it is a descendent. For example, a "group" defines
// both a type and a field in a single declaration. Thus, the locations
// corresponding to the type and field and their components will overlap.
// - Code which tries to interpret locations should probably be designed to
// ignore those that it doesn't understand, as more types of locations could
// be recorded in the future.
repeated Location location = 1;
message Location {
// Identifies which part of the FileDescriptorProto was defined at this
// location.
//
// Each element is a field number or an index. They form a path from
// the root FileDescriptorProto to the place where the definition. For
// example, this path:
// [ 4, 3, 2, 7, 1 ]
// refers to:
// file.message_type(3) // 4, 3
// .field(7) // 2, 7
// .name() // 1
// This is because FileDescriptorProto.message_type has field number 4:
// repeated DescriptorProto message_type = 4;
// and DescriptorProto.field has field number 2:
// repeated FieldDescriptorProto field = 2;
// and FieldDescriptorProto.name has field number 1:
// optional string name = 1;
//
// Thus, the above path gives the location of a field name. If we removed
// the last element:
// [ 4, 3, 2, 7 ]
// this path refers to the whole field declaration (from the beginning
// of the label to the terminating semicolon).
repeated int32 path = 1 [packed=true];
// Always has exactly three or four elements: start line, start column,
// end line (optional, otherwise assumed same as start line), end column.
// These are packed into a single field for efficiency. Note that line
// and column numbers are zero-based -- typically you will want to add
// 1 to each before displaying to a user.
repeated int32 span = 2 [packed=true];
// If this SourceCodeInfo represents a complete declaration, these are any
// comments appearing before and after the declaration which appear to be
// attached to the declaration.
//
// A series of line comments appearing on consecutive lines, with no other
// tokens appearing on those lines, will be treated as a single comment.
//
// leading_detached_comments will keep paragraphs of comments that appear
// before (but not connected to) the current element. Each paragraph,
// separated by empty lines, will be one comment element in the repeated
// field.
//
// Only the comment content is provided; comment markers (e.g. //) are
// stripped out. For block comments, leading whitespace and an asterisk
// will be stripped from the beginning of each line other than the first.
// Newlines are included in the output.
//
// Examples:
//
// optional int32 foo = 1; // Comment attached to foo.
// // Comment attached to bar.
// optional int32 bar = 2;
//
// optional string baz = 3;
// // Comment attached to baz.
// // Another line attached to baz.
//
// // Comment attached to qux.
// //
// // Another line attached to qux.
// optional double qux = 4;
//
// // Detached comment for corge. This is not leading or trailing comments
// // to qux or corge because there are blank lines separating it from
// // both.
//
// // Detached comment for corge paragraph 2.
//
// optional string corge = 5;
// /* Block comment attached
// * to corge. Leading asterisks
// * will be removed. */
// /* Block comment attached to
// * grault. */
// optional int32 grault = 6;
//
// // ignored detached comments.
optional string leading_comments = 3;
optional string trailing_comments = 4;
repeated string leading_detached_comments = 6;
}
}

View file

@ -1,11 +0,0 @@
Copyright 2014 Reverb Technologies, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at [apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View file

@ -1,24 +0,0 @@
# Readme
URL: https://github.com/swagger-api/swagger-ui/tree/master/dist
License: Apache License, Version 2.0
License File: LICENSE
## Description
Files from dist folder of https://github.com/swagger-api/swagger-ui.
These are dependency-free collection of HTML, Javascript, and CSS assets that
dynamically generate beautiful documentation and sandbox from a
Swagger-compliant API.
Instructions on how to use these:
https://github.com/swagger-api/swagger-ui#how-to-use-it
## Local Modifications
- Updated the url in index.html to "../../swaggerapi" as per instructions at:
https://github.com/swagger-api/swagger-ui#how-to-use-it
- Modified swagger-ui.js to list resources and operations in sorted order: https://github.com/kubernetes/kubernetes/pull/3421
- Set supportedSubmitMethods: [] in index.html to remove "Try it out" buttons.
- Remove the url query param to fix XSS issue:
https://github.com/kubernetes/kubernetes/pull/23234
LICENSE file has been created for compliance purposes.
Not included in original distribution.

View file

@ -1,125 +0,0 @@
/* http://meyerweb.com/eric/tools/css/reset/ v2.0 | 20110126 */
html,
body,
div,
span,
applet,
object,
iframe,
h1,
h2,
h3,
h4,
h5,
h6,
p,
blockquote,
pre,
a,
abbr,
acronym,
address,
big,
cite,
code,
del,
dfn,
em,
img,
ins,
kbd,
q,
s,
samp,
small,
strike,
strong,
sub,
sup,
tt,
var,
b,
u,
i,
center,
dl,
dt,
dd,
ol,
ul,
li,
fieldset,
form,
label,
legend,
table,
caption,
tbody,
tfoot,
thead,
tr,
th,
td,
article,
aside,
canvas,
details,
embed,
figure,
figcaption,
footer,
header,
hgroup,
menu,
nav,
output,
ruby,
section,
summary,
time,
mark,
audio,
video {
margin: 0;
padding: 0;
border: 0;
font-size: 100%;
font: inherit;
vertical-align: baseline;
}
/* HTML5 display-role reset for older browsers */
article,
aside,
details,
figcaption,
figure,
footer,
header,
hgroup,
menu,
nav,
section {
display: block;
}
body {
line-height: 1;
}
ol,
ul {
list-style: none;
}
blockquote,
q {
quotes: none;
}
blockquote:before,
blockquote:after,
q:before,
q:after {
content: '';
content: none;
}
table {
border-collapse: collapse;
border-spacing: 0;
}

File diff suppressed because it is too large Load diff

View file

@ -1,26 +0,0 @@
/* droid-sans-regular - latin */
@font-face {
font-family: 'Droid Sans';
font-style: normal;
font-weight: 400;
src: url('../fonts/droid-sans-v6-latin-regular.eot'); /* IE9 Compat Modes */
src: local('Droid Sans'), local('DroidSans'),
url('../fonts/droid-sans-v6-latin-regular.eot?#iefix') format('embedded-opentype'), /* IE6-IE8 */
url('../fonts/droid-sans-v6-latin-regular.woff2') format('woff2'), /* Super Modern Browsers */
url('../fonts/droid-sans-v6-latin-regular.woff') format('woff'), /* Modern Browsers */
url('../fonts/droid-sans-v6-latin-regular.ttf') format('truetype'), /* Safari, Android, iOS */
url('../fonts/droid-sans-v6-latin-regular.svg#DroidSans') format('svg'); /* Legacy iOS */
}
/* droid-sans-700 - latin */
@font-face {
font-family: 'Droid Sans';
font-style: normal;
font-weight: 700;
src: url('../fonts/droid-sans-v6-latin-700.eot'); /* IE9 Compat Modes */
src: local('Droid Sans Bold'), local('DroidSans-Bold'),
url('../fonts/droid-sans-v6-latin-700.eot?#iefix') format('embedded-opentype'), /* IE6-IE8 */
url('../fonts/droid-sans-v6-latin-700.woff2') format('woff2'), /* Super Modern Browsers */
url('../fonts/droid-sans-v6-latin-700.woff') format('woff'), /* Modern Browsers */
url('../fonts/droid-sans-v6-latin-700.ttf') format('truetype'), /* Safari, Android, iOS */
url('../fonts/droid-sans-v6-latin-700.svg#DroidSans') format('svg'); /* Legacy iOS */
}

Some files were not shown because too many files have changed in this diff Show more