Merged in feature/s3integration (pull request #47)

Set Up S3 integration * cfginterfaceandfirsts3funcs * addlocalstack * generallypassesfullsuite * addedmultipleattemptedpings * cleanup * stabiliseplusskip
2025-02-05 12:52:41 +00:00
parent 9254b91d45
commit 92334ad1dd
838 changed files with 139249 additions and 6671 deletions
@@ -1,20 +1,20 @@
 env:
  CIRRUS_CLONE_DEPTH: 1
-  GO_VERSION: go1.21.3
-
-freebsd_12_task:
-  freebsd_instance:
-    image_family: freebsd-12-3
-  install_script: |
-    pkg install -y go
-    GOBIN=$PWD/bin go install golang.org/dl/${GO_VERSION}@latest
-    bin/${GO_VERSION} download
-  build_script: bin/${GO_VERSION} build -v ./...
-  test_script: bin/${GO_VERSION} test -race ./...
+  GO_VERSION: go1.22.2

 freebsd_13_task:
  freebsd_instance:
-    image_family: freebsd-13-0
+    image_family: freebsd-13-2
+  install_script: |
+    pkg install -y go
+    GOBIN=$PWD/bin go install golang.org/dl/${GO_VERSION}@latest
+    bin/${GO_VERSION} download
+  build_script: bin/${GO_VERSION} build -v ./...
+  test_script: bin/${GO_VERSION} test -race ./...
+
+freebsd_14_task:
+  freebsd_instance:
+    image_family: freebsd-14-0
  install_script: |
    pkg install -y go
    GOBIN=$PWD/bin go install golang.org/dl/${GO_VERSION}@latest
@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.

 //go:build darwin || dragonfly || freebsd || netbsd || openbsd
-// +build darwin dragonfly freebsd netbsd openbsd

 package sysconf

@@ -16,6 +16,10 @@ const (
 	_HOST_NAME_MAX  = _MAXHOSTNAMELEN - 1
 	_LOGIN_NAME_MAX = _MAXLOGNAME
 	_SYMLOOP_MAX    = _MAXSYMLINKS
+
+	// _PTHREAD_STACK_MIN changed in macOS 14
+	_PTHREAD_STACK_MIN_LT_MACOS14 = 0x2000
+	_PTHREAD_STACK_MIN_GE_MACOS14 = 0x4000
 )

 var uname struct {
@@ -23,6 +27,21 @@ var uname struct {
 	macOSMajor int
 }

+func getMacOSMajor() int {
+	uname.Once.Do(func() {
+		var u unix.Utsname
+		err := unix.Uname(&u)
+		if err != nil {
+			return
+		}
+		rel := unix.ByteSliceToString(u.Release[:])
+		ver := strings.Split(rel, ".")
+		maj, _ := strconv.Atoi(ver[0])
+		uname.macOSMajor = maj
+	})
+	return uname.macOSMajor
+}
+
 // sysconf implements sysconf(4) as in the Darwin libc (derived from the FreeBSD
 // libc), version 1534.81.1.
 // See https://github.com/apple-oss-distributions/Libc/tree/Libc-1534.81.1.
@@ -91,7 +110,10 @@ func sysconf(name int) (int64, error) {
 	case SC_THREAD_PRIO_PROTECT:
 		return _POSIX_THREAD_PRIO_PROTECT, nil
 	case SC_THREAD_STACK_MIN:
-		return _PTHREAD_STACK_MIN, nil
+		if getMacOSMajor() < 23 {
+			return _PTHREAD_STACK_MIN_LT_MACOS14, nil
+		}
+		return _PTHREAD_STACK_MIN_GE_MACOS14, nil
 	case SC_THREAD_THREADS_MAX:
 		return -1, nil
 	case SC_TIMER_MAX:
@@ -140,18 +162,7 @@ func sysconf(name int) (int64, error) {
 		}
 		return _POSIX_SEMAPHORES, nil
 	case SC_SPAWN:
-		uname.Once.Do(func() {
-			var u unix.Utsname
-			err := unix.Uname(&u)
-			if err != nil {
-				return
-			}
-			rel := unix.ByteSliceToString(u.Release[:])
-			ver := strings.Split(rel, ".")
-			maj, _ := strconv.Atoi(ver[0])
-			uname.macOSMajor = maj
-		})
-		if uname.macOSMajor < 22 {
+		if getMacOSMajor() < 22 {
 			return -1, nil
 		}
 		// macOS 13 (Ventura) and later
@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.

 //go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd
-// +build darwin dragonfly freebsd linux netbsd openbsd

 package sysconf

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.

 //go:build darwin || dragonfly || freebsd || linux || openbsd
-// +build darwin dragonfly freebsd linux openbsd

 package sysconf

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.

 //go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris
-// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_defs_darwin.go

 //go:build darwin
-// +build darwin

 package sysconf

@@ -235,7 +234,6 @@ const (

 	_PTHREAD_DESTRUCTOR_ITERATIONS = 0x4
 	_PTHREAD_KEYS_MAX              = 0x200
-	_PTHREAD_STACK_MIN             = 0x2000
 )

 const (
@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_defs_dragonfly.go

 //go:build dragonfly
-// +build dragonfly

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_defs_freebsd.go

 //go:build freebsd
-// +build freebsd

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_defs_linux.go

 //go:build linux
-// +build linux

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_defs_netbsd.go

 //go:build netbsd
-// +build netbsd

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_defs_openbsd.go

 //go:build openbsd
-// +build openbsd

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_defs_solaris.go

 //go:build solaris
-// +build solaris

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_freebsd.go

 //go:build freebsd && 386
-// +build freebsd,386

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_freebsd.go

 //go:build freebsd && amd64
-// +build freebsd,amd64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_freebsd.go

 //go:build freebsd && arm
-// +build freebsd,arm

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_freebsd.go

 //go:build freebsd && arm64
-// +build freebsd,arm64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_freebsd.go

 //go:build freebsd && riscv64
-// +build freebsd,riscv64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && 386
-// +build linux,386

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && amd64
-// +build linux,amd64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && arm
-// +build linux,arm

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && arm64
-// +build linux,arm64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && loong64
-// +build linux,loong64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && mips
-// +build linux,mips

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && mips64
-// +build linux,mips64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && mips64le
-// +build linux,mips64le

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && mipsle
-// +build linux,mipsle

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && ppc64
-// +build linux,ppc64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && ppc64le
-// +build linux,ppc64le

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && riscv64
-// +build linux,riscv64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_linux.go

 //go:build linux && s390x
-// +build linux,s390x

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_netbsd.go

 //go:build netbsd && 386
-// +build netbsd,386

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_netbsd.go

 //go:build netbsd && amd64
-// +build netbsd,amd64

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_netbsd.go

 //go:build netbsd && arm
-// +build netbsd,arm

 package sysconf

@@ -2,7 +2,6 @@
 // cgo -godefs sysconf_values_netbsd.go

 //go:build netbsd && arm64
-// +build netbsd,arm64

 package sysconf

@@ -1,13 +1,23 @@
 env:
  CIRRUS_CLONE_DEPTH: 1
-  GO_VERSION: go1.21.3
+  GO_VERSION: go1.23.0

-freebsd_12_task:
+freebsd_13_task:
  freebsd_instance:
-    image_family: freebsd-12-3
+    image_family: freebsd-13-3
  install_script: |
    pkg install -y go
    GOBIN=$PWD/bin go install golang.org/dl/${GO_VERSION}@latest
    bin/${GO_VERSION} download
-  build_script: bin/${GO_VERSION} build -buildvcs=false -v ./...
-  test_script: bin/${GO_VERSION} test -buildvcs=false -race ./...
+  build_script: bin/${GO_VERSION} build -v ./...
+  test_script: bin/${GO_VERSION} test -race ./...
+
+freebsd_14_task:
+  freebsd_instance:
+    image_family: freebsd-14-0
+  install_script: |
+    pkg install -y go
+    GOBIN=$PWD/bin go install golang.org/dl/${GO_VERSION}@latest
+    bin/${GO_VERSION} download
+  build_script: bin/${GO_VERSION} build -v ./...
+  test_script: bin/${GO_VERSION} test -race ./...
@@ -73,3 +73,26 @@ func GetPossible() (int, error) {
 func GetPresent() (int, error) {
 	return getPresent()
 }
+
+// ListOffline returns the list of offline CPUs. See [GetOffline] for details on
+// when a CPU is considered offline.
+func ListOffline() ([]int, error) {
+	return listOffline()
+}
+
+// ListOnline returns the list of CPUs that are online and being scheduled.
+func ListOnline() ([]int, error) {
+	return listOnline()
+}
+
+// ListPossible returns the list of possible CPUs. See [GetPossible] for
+// details on when a CPU is considered possible.
+func ListPossible() ([]int, error) {
+	return listPossible()
+}
+
+// ListPresent returns the list of present CPUs. See [GetPresent] for
+// details on when a CPU is considered present.
+func ListPresent() ([]int, error) {
+	return listPresent()
+}
@@ -13,7 +13,6 @@
 // limitations under the License.

 //go:build darwin || dragonfly || freebsd || netbsd || openbsd
-// +build darwin dragonfly freebsd netbsd openbsd

 package numcpus

@@ -15,6 +15,7 @@
 package numcpus

 import (
+	"fmt"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -23,7 +24,14 @@ import (
 	"golang.org/x/sys/unix"
 )

-const sysfsCPUBasePath = "/sys/devices/system/cpu"
+const (
+	sysfsCPUBasePath = "/sys/devices/system/cpu"
+
+	offline  = "offline"
+	online   = "online"
+	possible = "possible"
+	present  = "present"
+)

 func getFromCPUAffinity() (int, error) {
 	var cpuSet unix.CPUSet
@@ -33,19 +41,26 @@ func getFromCPUAffinity() (int, error) {
 	return cpuSet.Count(), nil
 }

-func readCPURange(file string) (int, error) {
+func readCPURangeWith[T any](file string, f func(cpus string) (T, error)) (T, error) {
+	var zero T
 	buf, err := os.ReadFile(filepath.Join(sysfsCPUBasePath, file))
 	if err != nil {
-		return 0, err
+		return zero, err
 	}
-	return parseCPURange(strings.Trim(string(buf), "\n "))
+	return f(strings.Trim(string(buf), "\n "))
 }

-func parseCPURange(cpus string) (int, error) {
+func countCPURange(cpus string) (int, error) {
+	// Treat empty file as valid. This might be the case if there are no offline CPUs in which
+	// case /sys/devices/system/cpu/offline is empty.
+	if cpus == "" {
+		return 0, nil
+	}
+
 	n := int(0)
 	for _, cpuRange := range strings.Split(cpus, ",") {
-		if len(cpuRange) == 0 {
-			continue
+		if cpuRange == "" {
+			return 0, fmt.Errorf("empty CPU range in CPU string %q", cpus)
 		}
 		from, to, found := strings.Cut(cpuRange, "-")
 		first, err := strconv.ParseUint(from, 10, 32)
@@ -60,11 +75,49 @@ func parseCPURange(cpus string) (int, error) {
 		if err != nil {
 			return 0, err
 		}
+		if last < first {
+			return 0, fmt.Errorf("last CPU in range (%d) less than first (%d)", last, first)
+		}
 		n += int(last - first + 1)
 	}
 	return n, nil
 }

+func listCPURange(cpus string) ([]int, error) {
+	// See comment in countCPURange.
+	if cpus == "" {
+		return []int{}, nil
+	}
+
+	list := []int{}
+	for _, cpuRange := range strings.Split(cpus, ",") {
+		if cpuRange == "" {
+			return nil, fmt.Errorf("empty CPU range in CPU string %q", cpus)
+		}
+		from, to, found := strings.Cut(cpuRange, "-")
+		first, err := strconv.ParseUint(from, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+		if !found {
+			// range containing a single element
+			list = append(list, int(first))
+			continue
+		}
+		last, err := strconv.ParseUint(to, 10, 32)
+		if err != nil {
+			return nil, err
+		}
+		if last < first {
+			return nil, fmt.Errorf("last CPU in range (%d) less than first (%d)", last, first)
+		}
+		for cpu := int(first); cpu <= int(last); cpu++ {
+			list = append(list, cpu)
+		}
+	}
+	return list, nil
+}
+
 func getConfigured() (int, error) {
 	d, err := os.Open(sysfsCPUBasePath)
 	if err != nil {
@@ -100,20 +153,36 @@ func getKernelMax() (int, error) {
 }

 func getOffline() (int, error) {
-	return readCPURange("offline")
+	return readCPURangeWith(offline, countCPURange)
 }

 func getOnline() (int, error) {
 	if n, err := getFromCPUAffinity(); err == nil {
 		return n, nil
 	}
-	return readCPURange("online")
+	return readCPURangeWith(online, countCPURange)
 }

 func getPossible() (int, error) {
-	return readCPURange("possible")
+	return readCPURangeWith(possible, countCPURange)
 }

 func getPresent() (int, error) {
-	return readCPURange("present")
+	return readCPURangeWith(present, countCPURange)
+}
+
+func listOffline() ([]int, error) {
+	return readCPURangeWith(offline, listCPURange)
+}
+
+func listOnline() ([]int, error) {
+	return readCPURangeWith(online, listCPURange)
+}
+
+func listPossible() ([]int, error) {
+	return readCPURangeWith(possible, listCPURange)
+}
+
+func listPresent() ([]int, error) {
+	return readCPURangeWith(present, listCPURange)
 }
@@ -0,0 +1,33 @@
+// Copyright 2024 Tobias Klauser
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//go:build !linux
+
+package numcpus
+
+func listOffline() ([]int, error) {
+	return nil, ErrNotSupported
+}
+
+func listOnline() ([]int, error) {
+	return nil, ErrNotSupported
+}
+
+func listPossible() ([]int, error) {
+	return nil, ErrNotSupported
+}
+
+func listPresent() ([]int, error) {
+	return nil, ErrNotSupported
+}
@@ -13,7 +13,6 @@
 // limitations under the License.

 //go:build solaris
-// +build solaris

 package numcpus

@@ -13,7 +13,6 @@
 // limitations under the License.

 //go:build !darwin && !dragonfly && !freebsd && !linux && !netbsd && !openbsd && !solaris && !windows
-// +build !darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows

 package numcpus