mirror of
https://github.com/rd-stuffs/msm-4.14.git
synced 2025-02-20 11:45:48 +08:00
a63dbc5467
clock_gettime(CLOCK_BOOTTIME,) slows down after significant accumulation of suspend time creating a large offset between it and CLOCK_MONOTONIC time. The __iter_div_u64_rem() is only for the usage of adding a few second+nanosecond times and saving cycles on more expensive remainder and division operations, but iterates one second at a time which quickly goes out of scale in CLOCK_BOOTTIME's case since it was specified as nanoseconds only. The fix is to split off seconds from the boot time and cap the nanoseconds so that __iter_div_u64_rem does not iterate. Signed-off-by: Mark Salyzyn <salyzyn@google.com> Bug: 72406285 Change-Id: Ia647ef1e76b7ba3b0c003028d4b3b955635adabb Signed-off-by: khusika <khusikadhamar@gmail.com> Signed-off-by: azrim <mirzaspc@gmail.com>
400 lines
8.4 KiB
C
400 lines
8.4 KiB
C
/*
|
|
* Userspace implementations of gettimeofday() and friends.
|
|
*
|
|
* Copyright (C) 2017 Cavium, Inc.
|
|
* Copyright (C) 2015 Mentor Graphics Corporation
|
|
* Copyright (C) 2012 ARM Limited
|
|
*
|
|
* This program is free software; you can redistribute it and/or modify
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
* published by the Free Software Foundation.
|
|
*
|
|
* This program is distributed in the hope that it will be useful,
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
* GNU General Public License for more details.
|
|
*
|
|
* You should have received a copy of the GNU General Public License
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
*
|
|
* Author: Will Deacon <will.deacon@arm.com>
|
|
* Rewriten from arch64 version into C by: Andrew Pinski <apinski@cavium.com>
|
|
* Reworked and rebased over arm version by: Mark Salyzyn <salyzyn@android.com>
|
|
*/
|
|
|
|
#include <asm/barrier.h>
|
|
#include <linux/compiler.h> /* for notrace */
|
|
#include <linux/math64.h> /* for __iter_div_u64_rem() */
|
|
#include <uapi/linux/time.h> /* for struct timespec */
|
|
|
|
#include "compiler.h"
|
|
#include "datapage.h"
|
|
|
|
#ifdef ARCH_PROVIDES_TIMER
|
|
DEFINE_FALLBACK(gettimeofday, struct timeval *, tv, struct timezone *, tz)
|
|
#endif
|
|
DEFINE_FALLBACK(clock_gettime, clockid_t, clock, struct timespec *, ts)
|
|
DEFINE_FALLBACK(clock_getres, clockid_t, clock, struct timespec *, ts)
|
|
|
|
static notrace u32 vdso_read_begin(const struct vdso_data *vd)
|
|
{
|
|
u32 seq;
|
|
|
|
do {
|
|
seq = READ_ONCE(vd->tb_seq_count);
|
|
|
|
if ((seq & 1) == 0)
|
|
break;
|
|
|
|
cpu_relax();
|
|
} while (true);
|
|
|
|
smp_rmb(); /* Pairs with second smp_wmb in update_vsyscall */
|
|
return seq;
|
|
}
|
|
|
|
static notrace int vdso_read_retry(const struct vdso_data *vd, u32 start)
|
|
{
|
|
u32 seq;
|
|
|
|
smp_rmb(); /* Pairs with first smp_wmb in update_vsyscall */
|
|
seq = READ_ONCE(vd->tb_seq_count);
|
|
return seq != start;
|
|
}
|
|
|
|
static notrace int do_realtime_coarse(const struct vdso_data *vd,
|
|
struct timespec *ts)
|
|
{
|
|
u32 seq;
|
|
|
|
do {
|
|
seq = vdso_read_begin(vd);
|
|
|
|
ts->tv_sec = vd->xtime_coarse_sec;
|
|
ts->tv_nsec = vd->xtime_coarse_nsec;
|
|
|
|
} while (vdso_read_retry(vd, seq));
|
|
|
|
return 0;
|
|
}
|
|
|
|
static notrace int do_monotonic_coarse(const struct vdso_data *vd,
|
|
struct timespec *ts)
|
|
{
|
|
struct timespec tomono;
|
|
u32 seq;
|
|
u64 nsec;
|
|
|
|
do {
|
|
seq = vdso_read_begin(vd);
|
|
|
|
ts->tv_sec = vd->xtime_coarse_sec;
|
|
ts->tv_nsec = vd->xtime_coarse_nsec;
|
|
|
|
tomono.tv_sec = vd->wtm_clock_sec;
|
|
tomono.tv_nsec = vd->wtm_clock_nsec;
|
|
|
|
} while (vdso_read_retry(vd, seq));
|
|
|
|
ts->tv_sec += tomono.tv_sec;
|
|
/* open coding timespec_add_ns */
|
|
ts->tv_sec += __iter_div_u64_rem(ts->tv_nsec + tomono.tv_nsec,
|
|
NSEC_PER_SEC, &nsec);
|
|
ts->tv_nsec = nsec;
|
|
|
|
return 0;
|
|
}
|
|
|
|
#ifdef ARCH_PROVIDES_TIMER
|
|
|
|
/*
|
|
* Returns the clock delta, in nanoseconds left-shifted by the clock
|
|
* shift.
|
|
*/
|
|
static notrace u64 get_clock_shifted_nsec(const u64 cycle_last,
|
|
const u32 mult,
|
|
const u64 mask)
|
|
{
|
|
u64 res;
|
|
|
|
isb();
|
|
/* Read the virtual counter. */
|
|
res = arch_vdso_read_counter();
|
|
|
|
res = res - cycle_last;
|
|
|
|
res &= mask;
|
|
return res * mult;
|
|
}
|
|
|
|
static notrace int do_realtime(const struct vdso_data *vd, struct timespec *ts)
|
|
{
|
|
u32 seq, mult, shift;
|
|
u64 nsec, cycle_last;
|
|
#ifdef ARCH_CLOCK_FIXED_MASK
|
|
static const u64 mask = ARCH_CLOCK_FIXED_MASK;
|
|
#else
|
|
u64 mask;
|
|
#endif
|
|
vdso_xtime_clock_sec_t sec;
|
|
|
|
do {
|
|
seq = vdso_read_begin(vd);
|
|
|
|
if (vd->use_syscall)
|
|
return -1;
|
|
|
|
cycle_last = vd->cs_cycle_last;
|
|
|
|
mult = vd->cs_mono_mult;
|
|
shift = vd->cs_shift;
|
|
#ifndef ARCH_CLOCK_FIXED_MASK
|
|
mask = vd->cs_mask;
|
|
#endif
|
|
|
|
sec = vd->xtime_clock_sec;
|
|
nsec = vd->xtime_clock_snsec;
|
|
|
|
} while (unlikely(vdso_read_retry(vd, seq)));
|
|
|
|
nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
|
|
nsec >>= shift;
|
|
/* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
|
|
ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
|
|
ts->tv_nsec = nsec;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static notrace int do_monotonic(const struct vdso_data *vd, struct timespec *ts)
|
|
{
|
|
u32 seq, mult, shift;
|
|
u64 nsec, cycle_last;
|
|
#ifdef ARCH_CLOCK_FIXED_MASK
|
|
static const u64 mask = ARCH_CLOCK_FIXED_MASK;
|
|
#else
|
|
u64 mask;
|
|
#endif
|
|
vdso_wtm_clock_nsec_t wtm_nsec;
|
|
__kernel_time_t sec;
|
|
|
|
do {
|
|
seq = vdso_read_begin(vd);
|
|
|
|
if (vd->use_syscall)
|
|
return -1;
|
|
|
|
cycle_last = vd->cs_cycle_last;
|
|
|
|
mult = vd->cs_mono_mult;
|
|
shift = vd->cs_shift;
|
|
#ifndef ARCH_CLOCK_FIXED_MASK
|
|
mask = vd->cs_mask;
|
|
#endif
|
|
|
|
sec = vd->xtime_clock_sec;
|
|
nsec = vd->xtime_clock_snsec;
|
|
|
|
sec += vd->wtm_clock_sec;
|
|
wtm_nsec = vd->wtm_clock_nsec;
|
|
|
|
} while (unlikely(vdso_read_retry(vd, seq)));
|
|
|
|
nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
|
|
nsec >>= shift;
|
|
nsec += wtm_nsec;
|
|
/* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
|
|
ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
|
|
ts->tv_nsec = nsec;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static notrace int do_monotonic_raw(const struct vdso_data *vd,
|
|
struct timespec *ts)
|
|
{
|
|
u32 seq, mult, shift;
|
|
u64 nsec, cycle_last;
|
|
#ifdef ARCH_CLOCK_FIXED_MASK
|
|
static const u64 mask = ARCH_CLOCK_FIXED_MASK;
|
|
#else
|
|
u64 mask;
|
|
#endif
|
|
vdso_raw_time_sec_t sec;
|
|
|
|
do {
|
|
seq = vdso_read_begin(vd);
|
|
|
|
if (vd->use_syscall)
|
|
return -1;
|
|
|
|
cycle_last = vd->cs_cycle_last;
|
|
|
|
mult = vd->cs_raw_mult;
|
|
shift = vd->cs_shift;
|
|
#ifndef ARCH_CLOCK_FIXED_MASK
|
|
mask = vd->cs_mask;
|
|
#endif
|
|
|
|
sec = vd->raw_time_sec;
|
|
nsec = vd->raw_time_nsec;
|
|
|
|
} while (unlikely(vdso_read_retry(vd, seq)));
|
|
|
|
nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
|
|
nsec >>= shift;
|
|
/* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
|
|
ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
|
|
ts->tv_nsec = nsec;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static notrace int do_boottime(const struct vdso_data *vd, struct timespec *ts)
|
|
{
|
|
u32 seq, mult, shift;
|
|
u64 nsec, cycle_last;
|
|
vdso_wtm_clock_nsec_t wtm_nsec;
|
|
#ifdef ARCH_CLOCK_FIXED_MASK
|
|
static const u64 mask = ARCH_CLOCK_FIXED_MASK;
|
|
#else
|
|
u64 mask;
|
|
#endif
|
|
__kernel_time_t sec;
|
|
|
|
do {
|
|
seq = vdso_read_begin(vd);
|
|
|
|
if (vd->use_syscall)
|
|
return -1;
|
|
|
|
cycle_last = vd->cs_cycle_last;
|
|
|
|
mult = vd->cs_mono_mult;
|
|
shift = vd->cs_shift;
|
|
#ifndef ARCH_CLOCK_FIXED_MASK
|
|
mask = vd->cs_mask;
|
|
#endif
|
|
|
|
sec = vd->xtime_clock_sec;
|
|
nsec = vd->xtime_clock_snsec;
|
|
|
|
sec += vd->wtm_clock_sec + vd->btm_sec;
|
|
wtm_nsec = vd->wtm_clock_nsec + vd->btm_nsec;
|
|
|
|
} while (unlikely(vdso_read_retry(vd, seq)));
|
|
|
|
nsec += get_clock_shifted_nsec(cycle_last, mult, mask);
|
|
nsec >>= shift;
|
|
nsec += wtm_nsec;
|
|
|
|
/* open coding timespec_add_ns to save a ts->tv_nsec = 0 */
|
|
ts->tv_sec = sec + __iter_div_u64_rem(nsec, NSEC_PER_SEC, &nsec);
|
|
ts->tv_nsec = nsec;
|
|
|
|
return 0;
|
|
}
|
|
|
|
#endif /* ARCH_PROVIDES_TIMER */
|
|
|
|
notrace int __vdso_clock_gettime(clockid_t clock, struct timespec *ts)
|
|
{
|
|
const struct vdso_data *vd = __get_datapage();
|
|
|
|
switch (clock) {
|
|
case CLOCK_REALTIME_COARSE:
|
|
do_realtime_coarse(vd, ts);
|
|
break;
|
|
case CLOCK_MONOTONIC_COARSE:
|
|
do_monotonic_coarse(vd, ts);
|
|
break;
|
|
#ifdef ARCH_PROVIDES_TIMER
|
|
case CLOCK_REALTIME:
|
|
if (do_realtime(vd, ts))
|
|
goto fallback;
|
|
break;
|
|
case CLOCK_MONOTONIC:
|
|
if (do_monotonic(vd, ts))
|
|
goto fallback;
|
|
break;
|
|
case CLOCK_MONOTONIC_RAW:
|
|
if (do_monotonic_raw(vd, ts))
|
|
goto fallback;
|
|
break;
|
|
case CLOCK_BOOTTIME:
|
|
if (do_boottime(vd, ts))
|
|
goto fallback;
|
|
break;
|
|
#endif
|
|
default:
|
|
goto fallback;
|
|
}
|
|
|
|
return 0;
|
|
fallback:
|
|
return clock_gettime_fallback(clock, ts);
|
|
}
|
|
|
|
#ifdef ARCH_PROVIDES_TIMER
|
|
notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz)
|
|
{
|
|
const struct vdso_data *vd = __get_datapage();
|
|
|
|
if (likely(tv != NULL)) {
|
|
struct timespec ts;
|
|
|
|
if (do_realtime(vd, &ts))
|
|
return gettimeofday_fallback(tv, tz);
|
|
|
|
tv->tv_sec = ts.tv_sec;
|
|
tv->tv_usec = ts.tv_nsec / 1000;
|
|
}
|
|
|
|
if (unlikely(tz != NULL)) {
|
|
tz->tz_minuteswest = vd->tz_minuteswest;
|
|
tz->tz_dsttime = vd->tz_dsttime;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
#endif
|
|
|
|
int __vdso_clock_getres(clockid_t clock, struct timespec *res)
|
|
{
|
|
long nsec;
|
|
|
|
switch (clock) {
|
|
case CLOCK_REALTIME_COARSE:
|
|
case CLOCK_MONOTONIC_COARSE:
|
|
nsec = LOW_RES_NSEC;
|
|
break;
|
|
#ifdef ARCH_PROVIDES_TIMER
|
|
case CLOCK_REALTIME:
|
|
case CLOCK_MONOTONIC:
|
|
case CLOCK_MONOTONIC_RAW:
|
|
case CLOCK_BOOTTIME:
|
|
nsec = MONOTONIC_RES_NSEC;
|
|
break;
|
|
#endif
|
|
default:
|
|
return clock_getres_fallback(clock, res);
|
|
}
|
|
|
|
if (likely(res != NULL)) {
|
|
res->tv_sec = 0;
|
|
res->tv_nsec = nsec;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
notrace time_t __vdso_time(time_t *t)
|
|
{
|
|
const struct vdso_data *vd = __get_datapage();
|
|
time_t result = READ_ONCE(vd->xtime_coarse_sec);
|
|
|
|
if (t)
|
|
*t = result;
|
|
return result;
|
|
}
|