Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use rotating log files #437

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
+38 −21
Open

Use rotating log files #437

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fdb094f
attempt1, not working
lucyleeow Jul 8, 2020
edfe9f8
rotate log
lucyleeow Jul 9, 2020
2f7d769
fix imports
lucyleeow Jul 9, 2020
c26ea14
fix import
lucyleeow Jul 9, 2020
1ddb162
typo
lucyleeow Jul 9, 2020
dc99a77
pep8
lucyleeow Jul 9, 2020
7e01d6d
missed one
lucyleeow Jul 9, 2020
0570fe2
lint
lucyleeow Jul 9, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 10 additions & 4 deletions ramp-engine/ramp_engine/aws/worker.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,26 @@
import logging
from logging.handlers import RotatingFileHandler
import os

from ..base import BaseWorker, _get_traceback
from . import api as aws


log_dir = "logs"
if not os.path.exists(log_dir):
os.makedirs(log_dir)

logger = logging.getLogger('RAMP-AWS')

log_file = 'aws_worker.log'
log_file = os.path.join(log_dir, 'aws_worker.log')
formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s') # noqa
fileHandler = logging.FileHandler(log_file, mode='a')
fileHandler.setFormatter(formatter)
file_handler = RotatingFileHandler(log_file, maxBytes=1000, backupCount=2)
file_handler.setFormatter(formatter)
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(formatter)
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

streamHandler -> stream_handler


logger.setLevel(logging.DEBUG)
logger.addHandler(fileHandler)
logger.addHandler(file_handler)
logger.addHandler(streamHandler)


Expand Down
21 changes: 13 additions & 8 deletions ramp-engine/ramp_engine/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,24 @@
import logging
from logging.handlers import RotatingFileHandler
import os
from abc import ABCMeta, abstractmethod

log_dir = "logs"
Copy link
Collaborator

@rth rth Oct 16, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can't we make this configurable somehow?
The issue is that this log folder will depend on the location from which we run a script as far as I understand (and also happens at import). It might be better to have a function that is called as necessary that takes as input a config object and returns this logger. Not sure how easy that would be though.

if not os.path.exists(log_dir):
os.makedirs(log_dir)

logger = logging.getLogger('RAMP-WORKER')

log_file = "worker.log"
log_file = os.path.join(log_dir, "worker.log")
formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s') # noqa
fileHandler = logging.FileHandler(log_file, mode='a')
fileHandler.setFormatter(formatter)
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(formatter)
file_handler = RotatingFileHandler(log_file, maxBytes=1000, backupCount=2)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)

logger.setLevel(logging.DEBUG)
logger.addHandler(fileHandler)
logger.addHandler(streamHandler)

logger.addHandler(file_handler)
logger.addHandler(stream_handler)

class BaseWorker(metaclass=ABCMeta):
"""Metaclass used to build a RAMP worker. Do not use this class directly.
Expand Down
19 changes: 12 additions & 7 deletions ramp-engine/ramp_engine/dispatcher.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import logging
from logging.handlers import RotatingFileHandler
import multiprocessing
import numbers
import os
Expand Down Expand Up @@ -30,18 +31,22 @@

from .local import CondaEnvWorker

log_dir = "logs"
if not os.path.exists(log_dir):
os.makedirs(log_dir)

logger = logging.getLogger('RAMP-DISPATCHER')

log_file = 'dispatcher.log'
log_file = os.path.join(log_dir, 'dispatcher.log')
formatter = logging.Formatter('%(asctime)s %(name)s %(levelname)s %(message)s') # noqa
fileHandler = logging.FileHandler(log_file, mode='a')
fileHandler.setFormatter(formatter)
streamHandler = logging.StreamHandler()
streamHandler.setFormatter(formatter)
file_handler = RotatingFileHandler(log_file, maxBytes=1000, backupCount=2)
file_handler.setFormatter(formatter)
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(formatter)

logger.setLevel(logging.DEBUG)
logger.addHandler(fileHandler)
logger.addHandler(streamHandler)
logger.addHandler(file_handler)
logger.addHandler(stream_handler)


class Dispatcher:
Expand Down